Examples with ObjectMapper org.opensearch.index.mapper.ObjectMapper used on opensource projects

Example 1 with ObjectMapper

use of org.opensearch.index.mapper.ObjectMapper in project OpenSearch by opensearch-project.

the class TransportFieldCapabilitiesIndexAction method shardOperation.

private FieldCapabilitiesIndexResponse shardOperation(final FieldCapabilitiesIndexRequest request) throws IOException {
    if (canMatchShard(request) == false) {
        return new FieldCapabilitiesIndexResponse(request.index(), Collections.emptyMap(), false);
    }
    ShardId shardId = request.shardId();
    MapperService mapperService = indicesService.indexServiceSafe(shardId.getIndex()).mapperService();
    Set<String> fieldNames = new HashSet<>();
    for (String field : request.fields()) {
        fieldNames.addAll(mapperService.simpleMatchToFullName(field));
    }
    Predicate<String> fieldPredicate = indicesService.getFieldFilter().apply(shardId.getIndexName());
    Map<String, IndexFieldCapabilities> responseMap = new HashMap<>();
    for (String field : fieldNames) {
        MappedFieldType ft = mapperService.fieldType(field);
        if (ft != null) {
            if (indicesService.isMetadataField(mapperService.getIndexSettings().getIndexVersionCreated(), field) || fieldPredicate.test(ft.name())) {
                IndexFieldCapabilities fieldCap = new IndexFieldCapabilities(field, ft.familyTypeName(), ft.isSearchable(), ft.isAggregatable(), ft.meta());
                responseMap.put(field, fieldCap);
            } else {
                continue;
            }
            // add nested and object fields
            int dotIndex = ft.name().lastIndexOf('.');
            while (dotIndex > -1) {
                String parentField = ft.name().substring(0, dotIndex);
                if (responseMap.containsKey(parentField)) {
                    // we added this path on another field already
                    break;
                }
                // checks if the parent field contains sub-fields
                if (mapperService.fieldType(parentField) == null) {
                    // no field type, it must be an object field
                    ObjectMapper mapper = mapperService.getObjectMapper(parentField);
                    String type = mapper.nested().isNested() ? "nested" : "object";
                    IndexFieldCapabilities fieldCap = new IndexFieldCapabilities(parentField, type, false, false, Collections.emptyMap());
                    responseMap.put(parentField, fieldCap);
                }
                dotIndex = parentField.lastIndexOf('.');
            }
        }
    }
    return new FieldCapabilitiesIndexResponse(request.index(), responseMap, true);
}

Example 2 with ObjectMapper

use of org.opensearch.index.mapper.ObjectMapper in project OpenSearch by opensearch-project.

the class FetchPhase method prepareNestedHitContext.

/**
 * Resets the provided {@link HitContext} with information on the current
 * nested document. This includes the following:
 *   - Adding an initial {@link SearchHit} instance.
 *   - Loading the document source, filtering it based on the nested document ID, then
 *     setting it on {@link SourceLookup}. This allows fetch subphases that use the hit
 *     context to access the preloaded source.
 */
@SuppressWarnings("unchecked")
private HitContext prepareNestedHitContext(SearchContext context, int nestedTopDocId, int rootDocId, Map<String, Set<String>> storedToRequestedFields, LeafReaderContext subReaderContext, CheckedBiConsumer<Integer, FieldsVisitor, IOException> storedFieldReader) throws IOException {
    // Also if highlighting is requested on nested documents we need to fetch the _source from the root document,
    // otherwise highlighting will attempt to fetch the _source from the nested doc, which will fail,
    // because the entire _source is only stored with the root document.
    boolean needSource = sourceRequired(context) || context.highlight() != null;
    Uid rootId;
    Map<String, Object> rootSourceAsMap = null;
    XContentType rootSourceContentType = null;
    int nestedDocId = nestedTopDocId - subReaderContext.docBase;
    if (context instanceof InnerHitsContext.InnerHitSubContext) {
        InnerHitsContext.InnerHitSubContext innerHitsContext = (InnerHitsContext.InnerHitSubContext) context;
        rootId = innerHitsContext.getRootId();
        if (needSource) {
            SourceLookup rootLookup = innerHitsContext.getRootLookup();
            rootSourceAsMap = rootLookup.loadSourceIfNeeded();
            rootSourceContentType = rootLookup.sourceContentType();
        }
    } else {
        FieldsVisitor rootFieldsVisitor = new FieldsVisitor(needSource);
        loadStoredFields(context.mapperService(), storedFieldReader, rootFieldsVisitor, rootDocId);
        rootFieldsVisitor.postProcess(context.mapperService());
        rootId = rootFieldsVisitor.uid();
        if (needSource) {
            if (rootFieldsVisitor.source() != null) {
                Tuple<XContentType, Map<String, Object>> tuple = XContentHelper.convertToMap(rootFieldsVisitor.source(), false);
                rootSourceAsMap = tuple.v2();
                rootSourceContentType = tuple.v1();
            } else {
                rootSourceAsMap = Collections.emptyMap();
            }
        }
    }
    Map<String, DocumentField> docFields = emptyMap();
    Map<String, DocumentField> metaFields = emptyMap();
    if (context.hasStoredFields() && !context.storedFieldsContext().fieldNames().isEmpty()) {
        FieldsVisitor nestedFieldsVisitor = new CustomFieldsVisitor(storedToRequestedFields.keySet(), false);
        loadStoredFields(context.mapperService(), storedFieldReader, nestedFieldsVisitor, nestedDocId);
        if (nestedFieldsVisitor.fields().isEmpty() == false) {
            docFields = new HashMap<>();
            metaFields = new HashMap<>();
            fillDocAndMetaFields(context, nestedFieldsVisitor, storedToRequestedFields, docFields, metaFields);
        }
    }
    DocumentMapper documentMapper = context.mapperService().documentMapper();
    ObjectMapper nestedObjectMapper = documentMapper.findNestedObjectMapper(nestedDocId, context, subReaderContext);
    assert nestedObjectMapper != null;
    SearchHit.NestedIdentity nestedIdentity = getInternalNestedIdentity(context, nestedDocId, subReaderContext, context.mapperService(), nestedObjectMapper);
    SearchHit hit = new SearchHit(nestedTopDocId, rootId.id(), nestedIdentity, docFields, metaFields);
    // Use a clean, fresh SourceLookup
    HitContext hitContext = new HitContext(hit, subReaderContext, nestedDocId, new SourceLookup());
    if (rootSourceAsMap != null && rootSourceAsMap.isEmpty() == false) {
        // Isolate the nested json array object that matches with nested hit and wrap it back into the same json
        // structure with the nested json array object being the actual content. The latter is important, so that
        // features like source filtering and highlighting work consistent regardless of whether the field points
        // to a json object array for consistency reasons on how we refer to fields
        Map<String, Object> nestedSourceAsMap = new HashMap<>();
        Map<String, Object> current = nestedSourceAsMap;
        for (SearchHit.NestedIdentity nested = nestedIdentity; nested != null; nested = nested.getChild()) {
            String nestedPath = nested.getField().string();
            current.put(nestedPath, new HashMap<>());
            Object extractedValue = XContentMapValues.extractValue(nestedPath, rootSourceAsMap);
            List<?> nestedParsedSource;
            if (extractedValue instanceof List) {
                // nested field has an array value in the _source
                nestedParsedSource = (List<?>) extractedValue;
            } else if (extractedValue instanceof Map) {
                // nested field has an object value in the _source. This just means the nested field has just one inner object,
                // which is valid, but uncommon.
                nestedParsedSource = Collections.singletonList(extractedValue);
            } else {
                throw new IllegalStateException("extracted source isn't an object or an array");
            }
            if ((nestedParsedSource.get(0) instanceof Map) == false && nestedObjectMapper.parentObjectMapperAreNested(context.mapperService()) == false) {
                // This is why only the first element of nestedParsedSource needs to be checked.
                throw new IllegalArgumentException("Cannot execute inner hits. One or more parent object fields of nested field [" + nestedObjectMapper.name() + "] are not nested. All parent fields need to be nested fields too");
            }
            rootSourceAsMap = (Map<String, Object>) nestedParsedSource.get(nested.getOffset());
            if (nested.getChild() == null) {
                current.put(nestedPath, rootSourceAsMap);
            } else {
                Map<String, Object> next = new HashMap<>();
                current.put(nestedPath, next);
                current = next;
            }
        }
        hitContext.sourceLookup().setSource(nestedSourceAsMap);
        hitContext.sourceLookup().setSourceContentType(rootSourceContentType);
    }
    return hitContext;
}

Example 3 with ObjectMapper

use of org.opensearch.index.mapper.ObjectMapper in project OpenSearch by opensearch-project.

the class FetchPhase method getInternalNestedIdentity.

private SearchHit.NestedIdentity getInternalNestedIdentity(SearchContext context, int nestedSubDocId, LeafReaderContext subReaderContext, MapperService mapperService, ObjectMapper nestedObjectMapper) throws IOException {
    int currentParent = nestedSubDocId;
    ObjectMapper nestedParentObjectMapper;
    ObjectMapper current = nestedObjectMapper;
    String originalName = nestedObjectMapper.name();
    SearchHit.NestedIdentity nestedIdentity = null;
    final IndexSettings indexSettings = context.getQueryShardContext().getIndexSettings();
    do {
        Query parentFilter;
        nestedParentObjectMapper = current.getParentObjectMapper(mapperService);
        if (nestedParentObjectMapper != null) {
            if (nestedParentObjectMapper.nested().isNested() == false) {
                current = nestedParentObjectMapper;
                continue;
            }
            parentFilter = nestedParentObjectMapper.nestedTypeFilter();
        } else {
            parentFilter = Queries.newNonNestedFilter(context.indexShard().indexSettings().getIndexVersionCreated());
        }
        Query childFilter = nestedObjectMapper.nestedTypeFilter();
        if (childFilter == null) {
            current = nestedParentObjectMapper;
            continue;
        }
        final Weight childWeight = context.searcher().createWeight(context.searcher().rewrite(childFilter), ScoreMode.COMPLETE_NO_SCORES, 1f);
        Scorer childScorer = childWeight.scorer(subReaderContext);
        if (childScorer == null) {
            current = nestedParentObjectMapper;
            continue;
        }
        DocIdSetIterator childIter = childScorer.iterator();
        BitSet parentBits = context.bitsetFilterCache().getBitSetProducer(parentFilter).getBitSet(subReaderContext);
        int offset = 0;
        /*
             * Starts from the previous parent and finds the offset of the
             * <code>nestedSubDocID</code> within the nested children. Nested documents
             * are indexed in the same order than in the source array so the offset
             * of the nested child is the number of nested document with the same parent
             * that appear before him.
             */
        int previousParent = parentBits.prevSetBit(currentParent);
        for (int docId = childIter.advance(previousParent + 1); docId < nestedSubDocId && docId != DocIdSetIterator.NO_MORE_DOCS; docId = childIter.nextDoc()) {
            offset++;
        }
        currentParent = nestedSubDocId;
        current = nestedObjectMapper = nestedParentObjectMapper;
        int currentPrefix = current == null ? 0 : current.name().length() + 1;
        nestedIdentity = new SearchHit.NestedIdentity(originalName.substring(currentPrefix), offset, nestedIdentity);
        if (current != null) {
            originalName = current.name();
        }
    } while (current != null);
    return nestedIdentity;
}

Example 4 with ObjectMapper

use of org.opensearch.index.mapper.ObjectMapper in project OpenSearch by opensearch-project.

the class SortBuilder method resolveNestedQuery.

private static Query resolveNestedQuery(QueryShardContext context, NestedSortBuilder nestedSort, Query parentQuery) throws IOException {
    if (nestedSort == null || nestedSort.getPath() == null) {
        return null;
    }
    String nestedPath = nestedSort.getPath();
    QueryBuilder nestedFilter = nestedSort.getFilter();
    NestedSortBuilder nestedNestedSort = nestedSort.getNestedSort();
    // verify our nested path
    ObjectMapper nestedObjectMapper = context.getObjectMapper(nestedPath);
    if (nestedObjectMapper == null) {
        throw new QueryShardException(context, "[nested] failed to find nested object under path [" + nestedPath + "]");
    }
    if (!nestedObjectMapper.nested().isNested()) {
        throw new QueryShardException(context, "[nested] nested object under path [" + nestedPath + "] is not of nested type");
    }
    ObjectMapper objectMapper = context.nestedScope().getObjectMapper();
    // get our child query, potentially applying a users filter
    Query childQuery;
    try {
        context.nestedScope().nextLevel(nestedObjectMapper);
        if (nestedFilter != null) {
            assert nestedFilter == Rewriteable.rewrite(nestedFilter, context) : "nested filter is not rewritten";
            if (parentQuery == null) {
                // this is for back-compat, original single level nested sorting never applied a nested type filter
                childQuery = nestedFilter.toQuery(context);
            } else {
                childQuery = Queries.filtered(nestedObjectMapper.nestedTypeFilter(), nestedFilter.toQuery(context));
            }
        } else {
            childQuery = nestedObjectMapper.nestedTypeFilter();
        }
    } finally {
        context.nestedScope().previousLevel();
    }
    // apply filters from the previous nested level
    if (parentQuery != null) {
        if (objectMapper != null) {
            childQuery = Queries.filtered(childQuery, new ToChildBlockJoinQuery(parentQuery, context.bitsetFilter(objectMapper.nestedTypeFilter())));
        }
    }
    // wrap up our parent and child and either process the next level of nesting or return
    if (nestedNestedSort != null) {
        try {
            context.nestedScope().nextLevel(nestedObjectMapper);
            return resolveNestedQuery(context, nestedNestedSort, childQuery);
        } finally {
            context.nestedScope().previousLevel();
        }
    } else {
        return childQuery;
    }
}

Example 5 with ObjectMapper

use of org.opensearch.index.mapper.ObjectMapper in project OpenSearch by opensearch-project.

the class SortBuilder method resolveNested.

protected static Nested resolveNested(QueryShardContext context, NestedSortBuilder nestedSort) throws IOException {
    final Query childQuery = resolveNestedQuery(context, nestedSort, null);
    if (childQuery == null) {
        return null;
    }
    final ObjectMapper objectMapper = context.nestedScope().getObjectMapper();
    final Query parentQuery;
    if (objectMapper == null) {
        parentQuery = Queries.newNonNestedFilter(context.indexVersionCreated());
    } else {
        parentQuery = objectMapper.nestedTypeFilter();
    }
    return new Nested(context.bitsetFilter(parentQuery), childQuery, nestedSort, context.searcher());
}