use of org.opensearch.index.fieldvisitor.CustomFieldsVisitor in project OpenSearch by opensearch-project.
the class FetchPhase method prepareNestedHitContext.
/**
* Resets the provided {@link HitContext} with information on the current
* nested document. This includes the following:
* - Adding an initial {@link SearchHit} instance.
* - Loading the document source, filtering it based on the nested document ID, then
* setting it on {@link SourceLookup}. This allows fetch subphases that use the hit
* context to access the preloaded source.
*/
@SuppressWarnings("unchecked")
private HitContext prepareNestedHitContext(SearchContext context, int nestedTopDocId, int rootDocId, Map<String, Set<String>> storedToRequestedFields, LeafReaderContext subReaderContext, CheckedBiConsumer<Integer, FieldsVisitor, IOException> storedFieldReader) throws IOException {
// Also if highlighting is requested on nested documents we need to fetch the _source from the root document,
// otherwise highlighting will attempt to fetch the _source from the nested doc, which will fail,
// because the entire _source is only stored with the root document.
boolean needSource = sourceRequired(context) || context.highlight() != null;
Uid rootId;
Map<String, Object> rootSourceAsMap = null;
XContentType rootSourceContentType = null;
int nestedDocId = nestedTopDocId - subReaderContext.docBase;
if (context instanceof InnerHitsContext.InnerHitSubContext) {
InnerHitsContext.InnerHitSubContext innerHitsContext = (InnerHitsContext.InnerHitSubContext) context;
rootId = innerHitsContext.getRootId();
if (needSource) {
SourceLookup rootLookup = innerHitsContext.getRootLookup();
rootSourceAsMap = rootLookup.loadSourceIfNeeded();
rootSourceContentType = rootLookup.sourceContentType();
}
} else {
FieldsVisitor rootFieldsVisitor = new FieldsVisitor(needSource);
loadStoredFields(context.mapperService(), storedFieldReader, rootFieldsVisitor, rootDocId);
rootFieldsVisitor.postProcess(context.mapperService());
rootId = rootFieldsVisitor.uid();
if (needSource) {
if (rootFieldsVisitor.source() != null) {
Tuple<XContentType, Map<String, Object>> tuple = XContentHelper.convertToMap(rootFieldsVisitor.source(), false);
rootSourceAsMap = tuple.v2();
rootSourceContentType = tuple.v1();
} else {
rootSourceAsMap = Collections.emptyMap();
}
}
}
Map<String, DocumentField> docFields = emptyMap();
Map<String, DocumentField> metaFields = emptyMap();
if (context.hasStoredFields() && !context.storedFieldsContext().fieldNames().isEmpty()) {
FieldsVisitor nestedFieldsVisitor = new CustomFieldsVisitor(storedToRequestedFields.keySet(), false);
loadStoredFields(context.mapperService(), storedFieldReader, nestedFieldsVisitor, nestedDocId);
if (nestedFieldsVisitor.fields().isEmpty() == false) {
docFields = new HashMap<>();
metaFields = new HashMap<>();
fillDocAndMetaFields(context, nestedFieldsVisitor, storedToRequestedFields, docFields, metaFields);
}
}
DocumentMapper documentMapper = context.mapperService().documentMapper();
ObjectMapper nestedObjectMapper = documentMapper.findNestedObjectMapper(nestedDocId, context, subReaderContext);
assert nestedObjectMapper != null;
SearchHit.NestedIdentity nestedIdentity = getInternalNestedIdentity(context, nestedDocId, subReaderContext, context.mapperService(), nestedObjectMapper);
SearchHit hit = new SearchHit(nestedTopDocId, rootId.id(), nestedIdentity, docFields, metaFields);
// Use a clean, fresh SourceLookup
HitContext hitContext = new HitContext(hit, subReaderContext, nestedDocId, new SourceLookup());
if (rootSourceAsMap != null && rootSourceAsMap.isEmpty() == false) {
// Isolate the nested json array object that matches with nested hit and wrap it back into the same json
// structure with the nested json array object being the actual content. The latter is important, so that
// features like source filtering and highlighting work consistent regardless of whether the field points
// to a json object array for consistency reasons on how we refer to fields
Map<String, Object> nestedSourceAsMap = new HashMap<>();
Map<String, Object> current = nestedSourceAsMap;
for (SearchHit.NestedIdentity nested = nestedIdentity; nested != null; nested = nested.getChild()) {
String nestedPath = nested.getField().string();
current.put(nestedPath, new HashMap<>());
Object extractedValue = XContentMapValues.extractValue(nestedPath, rootSourceAsMap);
List<?> nestedParsedSource;
if (extractedValue instanceof List) {
// nested field has an array value in the _source
nestedParsedSource = (List<?>) extractedValue;
} else if (extractedValue instanceof Map) {
// nested field has an object value in the _source. This just means the nested field has just one inner object,
// which is valid, but uncommon.
nestedParsedSource = Collections.singletonList(extractedValue);
} else {
throw new IllegalStateException("extracted source isn't an object or an array");
}
if ((nestedParsedSource.get(0) instanceof Map) == false && nestedObjectMapper.parentObjectMapperAreNested(context.mapperService()) == false) {
// This is why only the first element of nestedParsedSource needs to be checked.
throw new IllegalArgumentException("Cannot execute inner hits. One or more parent object fields of nested field [" + nestedObjectMapper.name() + "] are not nested. All parent fields need to be nested fields too");
}
rootSourceAsMap = (Map<String, Object>) nestedParsedSource.get(nested.getOffset());
if (nested.getChild() == null) {
current.put(nestedPath, rootSourceAsMap);
} else {
Map<String, Object> next = new HashMap<>();
current.put(nestedPath, next);
current = next;
}
}
hitContext.sourceLookup().setSource(nestedSourceAsMap);
hitContext.sourceLookup().setSourceContentType(rootSourceContentType);
}
return hitContext;
}
use of org.opensearch.index.fieldvisitor.CustomFieldsVisitor in project OpenSearch by opensearch-project.
the class HighlightUtils method loadFieldValues.
/**
* Load field values for highlighting.
*/
public static List<Object> loadFieldValues(MappedFieldType fieldType, QueryShardContext context, FetchSubPhase.HitContext hitContext, boolean forceSource) throws IOException {
if (forceSource == false && fieldType.isStored()) {
CustomFieldsVisitor fieldVisitor = new CustomFieldsVisitor(singleton(fieldType.name()), false);
hitContext.reader().document(hitContext.docId(), fieldVisitor);
List<Object> textsToHighlight = fieldVisitor.fields().get(fieldType.name());
return textsToHighlight != null ? textsToHighlight : Collections.emptyList();
}
ValueFetcher fetcher = fieldType.valueFetcher(context, null, null);
return fetcher.fetchValues(hitContext.sourceLookup());
}
use of org.opensearch.index.fieldvisitor.CustomFieldsVisitor in project OpenSearch by opensearch-project.
the class StoredNumericValuesTests method testBytesAndNumericRepresentation.
public void testBytesAndNumericRepresentation() throws Exception {
IndexWriter writer = new IndexWriter(new ByteBuffersDirectory(), new IndexWriterConfig(Lucene.STANDARD_ANALYZER));
String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties").startObject("field1").field("type", "byte").field("store", true).endObject().startObject("field2").field("type", "short").field("store", true).endObject().startObject("field3").field("type", "integer").field("store", true).endObject().startObject("field4").field("type", "float").field("store", true).endObject().startObject("field5").field("type", "long").field("store", true).endObject().startObject("field6").field("type", "double").field("store", true).endObject().startObject("field7").field("type", "ip").field("store", true).endObject().startObject("field8").field("type", "ip").field("store", true).endObject().startObject("field9").field("type", "date").field("store", true).endObject().startObject("field10").field("type", "boolean").field("store", true).endObject().endObject().endObject().endObject());
MapperService mapperService = createIndex("test").mapperService();
DocumentMapper mapper = mapperService.merge("type", new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE);
ParsedDocument doc = mapper.parse(new SourceToParse("test", "type", "1", BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field("field1", 1).field("field2", 1).field("field3", 1).field("field4", 1.1).startArray("field5").value(1).value(2).value(3).endArray().field("field6", 1.1).field("field7", "192.168.1.1").field("field8", "2001:db8::2:1").field("field9", "2016-04-05").field("field10", true).endObject()), XContentType.JSON));
writer.addDocument(doc.rootDoc());
DirectoryReader reader = DirectoryReader.open(writer);
IndexSearcher searcher = new IndexSearcher(reader);
Set<String> fieldNames = Sets.newHashSet("field1", "field2", "field3", "field4", "field5", "field6", "field7", "field8", "field9", "field10");
CustomFieldsVisitor fieldsVisitor = new CustomFieldsVisitor(fieldNames, false);
searcher.doc(0, fieldsVisitor);
fieldsVisitor.postProcess(mapperService);
assertThat(fieldsVisitor.fields().size(), equalTo(10));
assertThat(fieldsVisitor.fields().get("field1").size(), equalTo(1));
assertThat(fieldsVisitor.fields().get("field1").get(0), equalTo((byte) 1));
assertThat(fieldsVisitor.fields().get("field2").size(), equalTo(1));
assertThat(fieldsVisitor.fields().get("field2").get(0), equalTo((short) 1));
assertThat(fieldsVisitor.fields().get("field3").size(), equalTo(1));
assertThat(fieldsVisitor.fields().get("field3").get(0), equalTo(1));
assertThat(fieldsVisitor.fields().get("field4").size(), equalTo(1));
assertThat(fieldsVisitor.fields().get("field4").get(0), equalTo(1.1f));
assertThat(fieldsVisitor.fields().get("field5").size(), equalTo(3));
assertThat(fieldsVisitor.fields().get("field5").get(0), equalTo(1L));
assertThat(fieldsVisitor.fields().get("field5").get(1), equalTo(2L));
assertThat(fieldsVisitor.fields().get("field5").get(2), equalTo(3L));
assertThat(fieldsVisitor.fields().get("field6").size(), equalTo(1));
assertThat(fieldsVisitor.fields().get("field6").get(0), equalTo(1.1));
assertThat(fieldsVisitor.fields().get("field7").size(), equalTo(1));
assertThat(fieldsVisitor.fields().get("field7").get(0), equalTo("192.168.1.1"));
assertThat(fieldsVisitor.fields().get("field8").size(), equalTo(1));
assertThat(fieldsVisitor.fields().get("field8").get(0), equalTo("2001:db8::2:1"));
assertThat(fieldsVisitor.fields().get("field9").size(), equalTo(1));
assertThat(fieldsVisitor.fields().get("field9").get(0), equalTo("2016-04-05T00:00:00.000Z"));
assertThat(fieldsVisitor.fields().get("field10").size(), equalTo(1));
assertThat(fieldsVisitor.fields().get("field10").get(0), equalTo(true));
reader.close();
writer.close();
}
Aggregations