Search in sources :

Example 1 with FieldsVisitor

use of org.elasticsearch.index.fieldvisitor.FieldsVisitor in project elasticsearch by elastic.

the class ShardGetService method innerGetLoadFromStoredFields.

private GetResult innerGetLoadFromStoredFields(String type, String id, String[] gFields, FetchSourceContext fetchSourceContext, Engine.GetResult get, MapperService mapperService) {
    Map<String, GetField> fields = null;
    BytesReference source = null;
    Versions.DocIdAndVersion docIdAndVersion = get.docIdAndVersion();
    FieldsVisitor fieldVisitor = buildFieldsVisitors(gFields, fetchSourceContext);
    if (fieldVisitor != null) {
        try {
            docIdAndVersion.context.reader().document(docIdAndVersion.docId, fieldVisitor);
        } catch (IOException e) {
            throw new ElasticsearchException("Failed to get type [" + type + "] and id [" + id + "]", e);
        }
        source = fieldVisitor.source();
        if (!fieldVisitor.fields().isEmpty()) {
            fieldVisitor.postProcess(mapperService);
            fields = new HashMap<>(fieldVisitor.fields().size());
            for (Map.Entry<String, List<Object>> entry : fieldVisitor.fields().entrySet()) {
                fields.put(entry.getKey(), new GetField(entry.getKey(), entry.getValue()));
            }
        }
    }
    DocumentMapper docMapper = mapperService.documentMapper(type);
    if (docMapper.parentFieldMapper().active()) {
        String parentId = ParentFieldSubFetchPhase.getParentId(docMapper.parentFieldMapper(), docIdAndVersion.context.reader(), docIdAndVersion.docId);
        if (fields == null) {
            fields = new HashMap<>(1);
        }
        fields.put(ParentFieldMapper.NAME, new GetField(ParentFieldMapper.NAME, Collections.singletonList(parentId)));
    }
    if (gFields != null && gFields.length > 0) {
        for (String field : gFields) {
            FieldMapper fieldMapper = docMapper.mappers().smartNameFieldMapper(field);
            if (fieldMapper == null) {
                if (docMapper.objectMappers().get(field) != null) {
                    // Only fail if we know it is a object field, missing paths / fields shouldn't fail.
                    throw new IllegalArgumentException("field [" + field + "] isn't a leaf field");
                }
            }
        }
    }
    if (!fetchSourceContext.fetchSource()) {
        source = null;
    } else if (fetchSourceContext.includes().length > 0 || fetchSourceContext.excludes().length > 0) {
        Map<String, Object> sourceAsMap;
        XContentType sourceContentType = null;
        // TODO: The source might parsed and available in the sourceLookup but that one uses unordered maps so different. Do we care?
        Tuple<XContentType, Map<String, Object>> typeMapTuple = XContentHelper.convertToMap(source, true);
        sourceContentType = typeMapTuple.v1();
        sourceAsMap = typeMapTuple.v2();
        sourceAsMap = XContentMapValues.filter(sourceAsMap, fetchSourceContext.includes(), fetchSourceContext.excludes());
        try {
            source = XContentFactory.contentBuilder(sourceContentType).map(sourceAsMap).bytes();
        } catch (IOException e) {
            throw new ElasticsearchException("Failed to get type [" + type + "] and id [" + id + "] with includes/excludes set", e);
        }
    }
    return new GetResult(shardId.getIndexName(), type, id, get.version(), get.exists(), source, fields);
}
Also used : BytesReference(org.elasticsearch.common.bytes.BytesReference) FieldsVisitor(org.elasticsearch.index.fieldvisitor.FieldsVisitor) CustomFieldsVisitor(org.elasticsearch.index.fieldvisitor.CustomFieldsVisitor) DocumentMapper(org.elasticsearch.index.mapper.DocumentMapper) IOException(java.io.IOException) ElasticsearchException(org.elasticsearch.ElasticsearchException) Versions(org.elasticsearch.common.lucene.uid.Versions) XContentType(org.elasticsearch.common.xcontent.XContentType) List(java.util.List) HashMap(java.util.HashMap) Map(java.util.Map) UidFieldMapper(org.elasticsearch.index.mapper.UidFieldMapper) FieldMapper(org.elasticsearch.index.mapper.FieldMapper) ParentFieldMapper(org.elasticsearch.index.mapper.ParentFieldMapper) SourceFieldMapper(org.elasticsearch.index.mapper.SourceFieldMapper) Tuple(org.elasticsearch.common.collect.Tuple)

Example 2 with FieldsVisitor

use of org.elasticsearch.index.fieldvisitor.FieldsVisitor in project elasticsearch by elastic.

the class FetchPhase method execute.

@Override
public void execute(SearchContext context) {
    final FieldsVisitor fieldsVisitor;
    Set<String> fieldNames = null;
    List<String> fieldNamePatterns = null;
    StoredFieldsContext storedFieldsContext = context.storedFieldsContext();
    if (storedFieldsContext == null) {
        // no fields specified, default to return source if no explicit indication
        if (!context.hasScriptFields() && !context.hasFetchSourceContext()) {
            context.fetchSourceContext(new FetchSourceContext(true));
        }
        fieldsVisitor = new FieldsVisitor(context.sourceRequested());
    } else if (storedFieldsContext.fetchFields() == false) {
        // disable stored fields entirely
        fieldsVisitor = null;
    } else {
        for (String fieldName : context.storedFieldsContext().fieldNames()) {
            if (fieldName.equals(SourceFieldMapper.NAME)) {
                FetchSourceContext fetchSourceContext = context.hasFetchSourceContext() ? context.fetchSourceContext() : FetchSourceContext.FETCH_SOURCE;
                context.fetchSourceContext(new FetchSourceContext(true, fetchSourceContext.includes(), fetchSourceContext.excludes()));
                continue;
            }
            if (Regex.isSimpleMatchPattern(fieldName)) {
                if (fieldNamePatterns == null) {
                    fieldNamePatterns = new ArrayList<>();
                }
                fieldNamePatterns.add(fieldName);
            } else {
                MappedFieldType fieldType = context.smartNameFieldType(fieldName);
                if (fieldType == null) {
                    // Only fail if we know it is a object field, missing paths / fields shouldn't fail.
                    if (context.getObjectMapper(fieldName) != null) {
                        throw new IllegalArgumentException("field [" + fieldName + "] isn't a leaf field");
                    }
                }
                if (fieldNames == null) {
                    fieldNames = new HashSet<>();
                }
                fieldNames.add(fieldName);
            }
        }
        boolean loadSource = context.sourceRequested();
        if (fieldNames == null && fieldNamePatterns == null) {
            // empty list specified, default to disable _source if no explicit indication
            fieldsVisitor = new FieldsVisitor(loadSource);
        } else {
            fieldsVisitor = new CustomFieldsVisitor(fieldNames == null ? Collections.emptySet() : fieldNames, fieldNamePatterns == null ? Collections.emptyList() : fieldNamePatterns, loadSource);
        }
    }
    SearchHit[] hits = new SearchHit[context.docIdsToLoadSize()];
    FetchSubPhase.HitContext hitContext = new FetchSubPhase.HitContext();
    for (int index = 0; index < context.docIdsToLoadSize(); index++) {
        if (context.isCancelled()) {
            throw new TaskCancelledException("cancelled");
        }
        int docId = context.docIdsToLoad()[context.docIdsToLoadFrom() + index];
        int readerIndex = ReaderUtil.subIndex(docId, context.searcher().getIndexReader().leaves());
        LeafReaderContext subReaderContext = context.searcher().getIndexReader().leaves().get(readerIndex);
        int subDocId = docId - subReaderContext.docBase;
        final SearchHit searchHit;
        try {
            int rootDocId = findRootDocumentIfNested(context, subReaderContext, subDocId);
            if (rootDocId != -1) {
                searchHit = createNestedSearchHit(context, docId, subDocId, rootDocId, fieldNames, fieldNamePatterns, subReaderContext);
            } else {
                searchHit = createSearchHit(context, fieldsVisitor, docId, subDocId, subReaderContext);
            }
        } catch (IOException e) {
            throw ExceptionsHelper.convertToElastic(e);
        }
        hits[index] = searchHit;
        hitContext.reset(searchHit, subReaderContext, subDocId, context.searcher());
        for (FetchSubPhase fetchSubPhase : fetchSubPhases) {
            fetchSubPhase.hitExecute(context, hitContext);
        }
    }
    for (FetchSubPhase fetchSubPhase : fetchSubPhases) {
        fetchSubPhase.hitsExecute(context, hits);
    }
    context.fetchResult().hits(new SearchHits(hits, context.queryResult().topDocs().totalHits, context.queryResult().topDocs().getMaxScore()));
}
Also used : FieldsVisitor(org.elasticsearch.index.fieldvisitor.FieldsVisitor) CustomFieldsVisitor(org.elasticsearch.index.fieldvisitor.CustomFieldsVisitor) SearchHit(org.elasticsearch.search.SearchHit) ArrayList(java.util.ArrayList) IOException(java.io.IOException) FetchSourceContext(org.elasticsearch.search.fetch.subphase.FetchSourceContext) CustomFieldsVisitor(org.elasticsearch.index.fieldvisitor.CustomFieldsVisitor) MappedFieldType(org.elasticsearch.index.mapper.MappedFieldType) InnerHitsFetchSubPhase(org.elasticsearch.search.fetch.subphase.InnerHitsFetchSubPhase) LeafReaderContext(org.apache.lucene.index.LeafReaderContext) TaskCancelledException(org.elasticsearch.tasks.TaskCancelledException) SearchHits(org.elasticsearch.search.SearchHits) HashSet(java.util.HashSet)

Example 3 with FieldsVisitor

use of org.elasticsearch.index.fieldvisitor.FieldsVisitor in project elasticsearch by elastic.

the class FetchPhase method createNestedSearchHit.

private SearchHit createNestedSearchHit(SearchContext context, int nestedTopDocId, int nestedSubDocId, int rootSubDocId, Set<String> fieldNames, List<String> fieldNamePatterns, LeafReaderContext subReaderContext) throws IOException {
    // Also if highlighting is requested on nested documents we need to fetch the _source from the root document,
    // otherwise highlighting will attempt to fetch the _source from the nested doc, which will fail,
    // because the entire _source is only stored with the root document.
    final FieldsVisitor rootFieldsVisitor = new FieldsVisitor(context.sourceRequested() || context.highlight() != null);
    loadStoredFields(context, subReaderContext, rootFieldsVisitor, rootSubDocId);
    rootFieldsVisitor.postProcess(context.mapperService());
    Map<String, SearchHitField> searchFields = getSearchFields(context, nestedSubDocId, fieldNames, fieldNamePatterns, subReaderContext);
    DocumentMapper documentMapper = context.mapperService().documentMapper(rootFieldsVisitor.uid().type());
    SourceLookup sourceLookup = context.lookup().source();
    sourceLookup.setSegmentAndDocument(subReaderContext, nestedSubDocId);
    ObjectMapper nestedObjectMapper = documentMapper.findNestedObjectMapper(nestedSubDocId, context, subReaderContext);
    assert nestedObjectMapper != null;
    SearchHit.NestedIdentity nestedIdentity = getInternalNestedIdentity(context, nestedSubDocId, subReaderContext, documentMapper, nestedObjectMapper);
    BytesReference source = rootFieldsVisitor.source();
    if (source != null) {
        Tuple<XContentType, Map<String, Object>> tuple = XContentHelper.convertToMap(source, true);
        Map<String, Object> sourceAsMap = tuple.v2();
        // Isolate the nested json array object that matches with nested hit and wrap it back into the same json
        // structure with the nested json array object being the actual content. The latter is important, so that
        // features like source filtering and highlighting work consistent regardless of whether the field points
        // to a json object array for consistency reasons on how we refer to fields
        Map<String, Object> nestedSourceAsMap = new HashMap<>();
        Map<String, Object> current = nestedSourceAsMap;
        for (SearchHit.NestedIdentity nested = nestedIdentity; nested != null; nested = nested.getChild()) {
            String nestedPath = nested.getField().string();
            current.put(nestedPath, new HashMap<>());
            Object extractedValue = XContentMapValues.extractValue(nestedPath, sourceAsMap);
            List<Map<String, Object>> nestedParsedSource;
            if (extractedValue instanceof List) {
                // nested field has an array value in the _source
                nestedParsedSource = (List<Map<String, Object>>) extractedValue;
            } else if (extractedValue instanceof Map) {
                // nested field has an object value in the _source. This just means the nested field has just one inner object, which is valid, but uncommon.
                nestedParsedSource = Collections.singletonList((Map<String, Object>) extractedValue);
            } else {
                throw new IllegalStateException("extracted source isn't an object or an array");
            }
            sourceAsMap = nestedParsedSource.get(nested.getOffset());
            if (nested.getChild() == null) {
                current.put(nestedPath, sourceAsMap);
            } else {
                Map<String, Object> next = new HashMap<>();
                current.put(nestedPath, next);
                current = next;
            }
        }
        context.lookup().source().setSource(nestedSourceAsMap);
        XContentType contentType = tuple.v1();
        BytesReference nestedSource = contentBuilder(contentType).map(sourceAsMap).bytes();
        context.lookup().source().setSource(nestedSource);
        context.lookup().source().setSourceContentType(contentType);
    }
    return new SearchHit(nestedTopDocId, rootFieldsVisitor.uid().id(), documentMapper.typeText(), nestedIdentity, searchFields);
}
Also used : BytesReference(org.elasticsearch.common.bytes.BytesReference) FieldsVisitor(org.elasticsearch.index.fieldvisitor.FieldsVisitor) CustomFieldsVisitor(org.elasticsearch.index.fieldvisitor.CustomFieldsVisitor) SourceLookup(org.elasticsearch.search.lookup.SourceLookup) SearchHit(org.elasticsearch.search.SearchHit) HashMap(java.util.HashMap) DocumentMapper(org.elasticsearch.index.mapper.DocumentMapper) XContentType(org.elasticsearch.common.xcontent.XContentType) SearchHitField(org.elasticsearch.search.SearchHitField) ArrayList(java.util.ArrayList) List(java.util.List) HashMap(java.util.HashMap) Map(java.util.Map) ObjectMapper(org.elasticsearch.index.mapper.ObjectMapper)

Example 4 with FieldsVisitor

use of org.elasticsearch.index.fieldvisitor.FieldsVisitor in project elasticsearch by elastic.

the class FetchPhase method getSearchFields.

private Map<String, SearchHitField> getSearchFields(SearchContext context, int nestedSubDocId, Set<String> fieldNames, List<String> fieldNamePatterns, LeafReaderContext subReaderContext) {
    Map<String, SearchHitField> searchFields = null;
    if (context.hasStoredFields() && !context.storedFieldsContext().fieldNames().isEmpty()) {
        FieldsVisitor nestedFieldsVisitor = new CustomFieldsVisitor(fieldNames == null ? Collections.emptySet() : fieldNames, fieldNamePatterns == null ? Collections.emptyList() : fieldNamePatterns, false);
        if (nestedFieldsVisitor != null) {
            loadStoredFields(context, subReaderContext, nestedFieldsVisitor, nestedSubDocId);
            nestedFieldsVisitor.postProcess(context.mapperService());
            if (!nestedFieldsVisitor.fields().isEmpty()) {
                searchFields = new HashMap<>(nestedFieldsVisitor.fields().size());
                for (Map.Entry<String, List<Object>> entry : nestedFieldsVisitor.fields().entrySet()) {
                    searchFields.put(entry.getKey(), new SearchHitField(entry.getKey(), entry.getValue()));
                }
            }
        }
    }
    return searchFields;
}
Also used : FieldsVisitor(org.elasticsearch.index.fieldvisitor.FieldsVisitor) CustomFieldsVisitor(org.elasticsearch.index.fieldvisitor.CustomFieldsVisitor) CustomFieldsVisitor(org.elasticsearch.index.fieldvisitor.CustomFieldsVisitor) SearchHitField(org.elasticsearch.search.SearchHitField) ArrayList(java.util.ArrayList) List(java.util.List) HashMap(java.util.HashMap) Map(java.util.Map)

Example 5 with FieldsVisitor

use of org.elasticsearch.index.fieldvisitor.FieldsVisitor in project elasticsearch by elastic.

the class SourceLookup method loadSourceIfNeeded.

private Map<String, Object> loadSourceIfNeeded() {
    if (source != null) {
        return source;
    }
    if (sourceAsBytes != null) {
        Tuple<XContentType, Map<String, Object>> tuple = sourceAsMapAndType(sourceAsBytes);
        sourceContentType = tuple.v1();
        source = tuple.v2();
        return source;
    }
    try {
        FieldsVisitor sourceFieldVisitor = new FieldsVisitor(true);
        reader.document(docId, sourceFieldVisitor);
        BytesReference source = sourceFieldVisitor.source();
        if (source == null) {
            this.source = emptyMap();
            this.sourceContentType = null;
        } else {
            Tuple<XContentType, Map<String, Object>> tuple = sourceAsMapAndType(source);
            this.sourceContentType = tuple.v1();
            this.source = tuple.v2();
        }
    } catch (Exception e) {
        throw new ElasticsearchParseException("failed to parse / load source", e);
    }
    return this.source;
}
Also used : BytesReference(org.elasticsearch.common.bytes.BytesReference) FieldsVisitor(org.elasticsearch.index.fieldvisitor.FieldsVisitor) XContentType(org.elasticsearch.common.xcontent.XContentType) ElasticsearchParseException(org.elasticsearch.ElasticsearchParseException) Collections.emptyMap(java.util.Collections.emptyMap) Map(java.util.Map) ElasticsearchParseException(org.elasticsearch.ElasticsearchParseException)

Aggregations

FieldsVisitor (org.elasticsearch.index.fieldvisitor.FieldsVisitor)5 Map (java.util.Map)4 CustomFieldsVisitor (org.elasticsearch.index.fieldvisitor.CustomFieldsVisitor)4 ArrayList (java.util.ArrayList)3 HashMap (java.util.HashMap)3 List (java.util.List)3 BytesReference (org.elasticsearch.common.bytes.BytesReference)3 XContentType (org.elasticsearch.common.xcontent.XContentType)3 IOException (java.io.IOException)2 DocumentMapper (org.elasticsearch.index.mapper.DocumentMapper)2 SearchHit (org.elasticsearch.search.SearchHit)2 SearchHitField (org.elasticsearch.search.SearchHitField)2 Collections.emptyMap (java.util.Collections.emptyMap)1 HashSet (java.util.HashSet)1 LeafReaderContext (org.apache.lucene.index.LeafReaderContext)1 ElasticsearchException (org.elasticsearch.ElasticsearchException)1 ElasticsearchParseException (org.elasticsearch.ElasticsearchParseException)1 Tuple (org.elasticsearch.common.collect.Tuple)1 Versions (org.elasticsearch.common.lucene.uid.Versions)1 FieldMapper (org.elasticsearch.index.mapper.FieldMapper)1