use of org.opensearch.index.mapper.Mapper in project k-NN by opensearch-project.
the class KNNVectorFieldMapperTests method testKNNVectorFieldMapper_merge_fromModel.
public void testKNNVectorFieldMapper_merge_fromModel() throws IOException {
String fieldName = "test-field-name";
String indexName = "test-index-name";
Settings settings = Settings.builder().put(settings(CURRENT).build()).build();
String modelId = "test-id";
int dimension = 133;
ModelDao mockModelDao = mock(ModelDao.class);
ModelMetadata mockModelMetadata = new ModelMetadata(KNNEngine.DEFAULT, SpaceType.DEFAULT, dimension, ModelState.CREATED, ZonedDateTime.now(ZoneOffset.UTC).toString(), "", "");
when(mockModelDao.getMetadata(modelId)).thenReturn(mockModelMetadata);
KNNVectorFieldMapper.TypeParser typeParser = new KNNVectorFieldMapper.TypeParser(() -> mockModelDao);
XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().field("type", "knn_vector").field(MODEL_ID, modelId).endObject();
KNNVectorFieldMapper.Builder builder = (KNNVectorFieldMapper.Builder) typeParser.parse(fieldName, xContentBuilderToMap(xContentBuilder), buildParserContext(indexName, settings));
Mapper.BuilderContext builderContext = new Mapper.BuilderContext(settings, new ContentPath());
KNNVectorFieldMapper knnVectorFieldMapper1 = builder.build(builderContext);
// merge with itself - should be successful
KNNVectorFieldMapper knnVectorFieldMapperMerge1 = (KNNVectorFieldMapper) knnVectorFieldMapper1.merge(knnVectorFieldMapper1);
assertEquals(knnVectorFieldMapper1.modelId, knnVectorFieldMapperMerge1.modelId);
// merge with another mapper of the same field with same context
KNNVectorFieldMapper knnVectorFieldMapper2 = builder.build(builderContext);
KNNVectorFieldMapper knnVectorFieldMapperMerge2 = (KNNVectorFieldMapper) knnVectorFieldMapper1.merge(knnVectorFieldMapper2);
assertEquals(knnVectorFieldMapper1.modelId, knnVectorFieldMapperMerge2.modelId);
// merge with another mapper of the same field with different context
xContentBuilder = XContentFactory.jsonBuilder().startObject().field("type", "knn_vector").field("dimension", dimension).startObject(KNN_METHOD).field(NAME, METHOD_HNSW).endObject().endObject();
builder = (KNNVectorFieldMapper.Builder) typeParser.parse(fieldName, xContentBuilderToMap(xContentBuilder), buildParserContext(indexName, settings));
KNNVectorFieldMapper knnVectorFieldMapper3 = builder.build(builderContext);
expectThrows(IllegalArgumentException.class, () -> knnVectorFieldMapper1.merge(knnVectorFieldMapper3));
}
use of org.opensearch.index.mapper.Mapper in project OpenSearch by opensearch-project.
the class TransportGetFieldMappingsIndexAction method shardOperation.
@Override
protected GetFieldMappingsResponse shardOperation(final GetFieldMappingsIndexRequest request, ShardId shardId) {
assert shardId != null;
IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex());
Version indexCreatedVersion = indexService.mapperService().getIndexSettings().getIndexVersionCreated();
Predicate<String> metadataFieldPredicate = (f) -> indicesService.isMetadataField(indexCreatedVersion, f);
Predicate<String> fieldPredicate = metadataFieldPredicate.or(indicesService.getFieldFilter().apply(shardId.getIndexName()));
DocumentMapper mapper = indexService.mapperService().documentMapper();
Collection<String> typeIntersection;
if (request.types().length == 0) {
typeIntersection = mapper == null ? Collections.emptySet() : Collections.singleton(mapper.type());
} else {
typeIntersection = mapper != null && Regex.simpleMatch(request.types(), mapper.type()) ? Collections.singleton(mapper.type()) : Collections.emptySet();
if (typeIntersection.isEmpty()) {
throw new TypeMissingException(shardId.getIndex(), request.types());
}
}
Map<String, Map<String, FieldMappingMetadata>> typeMappings = new HashMap<>();
for (String type : typeIntersection) {
DocumentMapper documentMapper = indexService.mapperService().documentMapper(type);
Map<String, FieldMappingMetadata> fieldMapping = findFieldMappingsByType(fieldPredicate, documentMapper, request);
if (!fieldMapping.isEmpty()) {
typeMappings.put(type, fieldMapping);
}
}
return new GetFieldMappingsResponse(singletonMap(shardId.getIndexName(), Collections.unmodifiableMap(typeMappings)));
}
use of org.opensearch.index.mapper.Mapper in project OpenSearch by opensearch-project.
the class ShardGetService method innerGetLoadFromStoredFields.
private GetResult innerGetLoadFromStoredFields(String id, String[] storedFields, FetchSourceContext fetchSourceContext, Engine.GetResult get, MapperService mapperService) {
assert get.exists() : "method should only be called if document could be retrieved";
// check first if stored fields to be loaded don't contain an object field
DocumentMapper docMapper = mapperService.documentMapper();
if (storedFields != null) {
for (String field : storedFields) {
Mapper fieldMapper = docMapper.mappers().getMapper(field);
if (fieldMapper == null) {
if (docMapper.objectMappers().get(field) != null) {
// Only fail if we know it is a object field, missing paths / fields shouldn't fail.
throw new IllegalArgumentException("field [" + field + "] isn't a leaf field");
}
}
}
}
Map<String, DocumentField> documentFields = null;
Map<String, DocumentField> metadataFields = null;
BytesReference source = null;
DocIdAndVersion docIdAndVersion = get.docIdAndVersion();
// force fetching source if we read from translog and need to recreate stored fields
boolean forceSourceForComputingTranslogStoredFields = get.isFromTranslog() && storedFields != null && Stream.of(storedFields).anyMatch(f -> TranslogLeafReader.ALL_FIELD_NAMES.contains(f) == false);
FieldsVisitor fieldVisitor = buildFieldsVisitors(storedFields, forceSourceForComputingTranslogStoredFields ? FetchSourceContext.FETCH_SOURCE : fetchSourceContext);
if (fieldVisitor != null) {
try {
docIdAndVersion.reader.document(docIdAndVersion.docId, fieldVisitor);
} catch (IOException e) {
throw new OpenSearchException("Failed to get id [" + id + "]", e);
}
source = fieldVisitor.source();
// in case we read from translog, some extra steps are needed to make _source consistent and to load stored fields
if (get.isFromTranslog()) {
// just make source consistent by reapplying source filters from mapping (possibly also nulling the source)
if (forceSourceForComputingTranslogStoredFields == false) {
try {
source = indexShard.mapperService().documentMapper().sourceMapper().applyFilters(source, null);
} catch (IOException e) {
throw new OpenSearchException("Failed to reapply filters for [" + id + "] after reading from translog", e);
}
} else {
// Slow path: recreate stored fields from original source
assert source != null : "original source in translog must exist";
SourceToParse sourceToParse = new SourceToParse(shardId.getIndexName(), MapperService.SINGLE_MAPPING_NAME, id, source, XContentHelper.xContentType(source), fieldVisitor.routing());
ParsedDocument doc = indexShard.mapperService().documentMapper().parse(sourceToParse);
assert doc.dynamicMappingsUpdate() == null : "mapping updates should not be required on already-indexed doc";
// update special fields
doc.updateSeqID(docIdAndVersion.seqNo, docIdAndVersion.primaryTerm);
doc.version().setLongValue(docIdAndVersion.version);
// retrieve stored fields from parsed doc
fieldVisitor = buildFieldsVisitors(storedFields, fetchSourceContext);
for (IndexableField indexableField : doc.rootDoc().getFields()) {
IndexableFieldType fieldType = indexableField.fieldType();
if (fieldType.stored()) {
FieldInfo fieldInfo = new FieldInfo(indexableField.name(), 0, false, false, false, IndexOptions.NONE, DocValuesType.NONE, -1, Collections.emptyMap(), 0, 0, 0, false);
StoredFieldVisitor.Status status = fieldVisitor.needsField(fieldInfo);
if (status == StoredFieldVisitor.Status.YES) {
if (indexableField.numericValue() != null) {
fieldVisitor.objectField(fieldInfo, indexableField.numericValue());
} else if (indexableField.binaryValue() != null) {
fieldVisitor.binaryField(fieldInfo, indexableField.binaryValue());
} else if (indexableField.stringValue() != null) {
fieldVisitor.objectField(fieldInfo, indexableField.stringValue());
}
} else if (status == StoredFieldVisitor.Status.STOP) {
break;
}
}
}
// retrieve source (with possible transformations, e.g. source filters
source = fieldVisitor.source();
}
}
// put stored fields into result objects
if (!fieldVisitor.fields().isEmpty()) {
fieldVisitor.postProcess(mapperService);
documentFields = new HashMap<>();
metadataFields = new HashMap<>();
for (Map.Entry<String, List<Object>> entry : fieldVisitor.fields().entrySet()) {
if (mapperService.isMetadataField(entry.getKey())) {
metadataFields.put(entry.getKey(), new DocumentField(entry.getKey(), entry.getValue()));
} else {
documentFields.put(entry.getKey(), new DocumentField(entry.getKey(), entry.getValue()));
}
}
}
}
if (source != null) {
// apply request-level source filtering
if (fetchSourceContext.fetchSource() == false) {
source = null;
} else if (fetchSourceContext.includes().length > 0 || fetchSourceContext.excludes().length > 0) {
Map<String, Object> sourceAsMap;
// TODO: The source might be parsed and available in the sourceLookup but that one uses unordered maps so different.
// Do we care?
Tuple<XContentType, Map<String, Object>> typeMapTuple = XContentHelper.convertToMap(source, true);
XContentType sourceContentType = typeMapTuple.v1();
sourceAsMap = typeMapTuple.v2();
sourceAsMap = XContentMapValues.filter(sourceAsMap, fetchSourceContext.includes(), fetchSourceContext.excludes());
try {
source = BytesReference.bytes(XContentFactory.contentBuilder(sourceContentType).map(sourceAsMap));
} catch (IOException e) {
throw new OpenSearchException("Failed to get id [" + id + "] with includes/excludes set", e);
}
}
}
if (!fetchSourceContext.fetchSource()) {
source = null;
}
if (source != null && get.isFromTranslog()) {
// reapply source filters from mapping (possibly also nulling the source)
try {
source = docMapper.sourceMapper().applyFilters(source, null);
} catch (IOException e) {
throw new OpenSearchException("Failed to reapply filters for [" + id + "] after reading from translog", e);
}
}
if (source != null && (fetchSourceContext.includes().length > 0 || fetchSourceContext.excludes().length > 0)) {
Map<String, Object> sourceAsMap;
// TODO: The source might parsed and available in the sourceLookup but that one uses unordered maps so different. Do we care?
Tuple<XContentType, Map<String, Object>> typeMapTuple = XContentHelper.convertToMap(source, true);
XContentType sourceContentType = typeMapTuple.v1();
sourceAsMap = typeMapTuple.v2();
sourceAsMap = XContentMapValues.filter(sourceAsMap, fetchSourceContext.includes(), fetchSourceContext.excludes());
try {
source = BytesReference.bytes(XContentFactory.contentBuilder(sourceContentType).map(sourceAsMap));
} catch (IOException e) {
throw new OpenSearchException("Failed to get id [" + id + "] with includes/excludes set", e);
}
}
return new GetResult(shardId.getIndexName(), id, get.docIdAndVersion().seqNo, get.docIdAndVersion().primaryTerm, get.version(), get.exists(), source, documentFields, metadataFields);
}
use of org.opensearch.index.mapper.Mapper in project OpenSearch by opensearch-project.
the class CategoryContextMappingTests method testIndexingWithMixedTypeContextList.
public void testIndexingWithMixedTypeContextList() throws Exception {
String mapping = Strings.toString(jsonBuilder().startObject().startObject("type1").startObject("properties").startObject("completion").field("type", "completion").startArray("contexts").startObject().field("name", "ctx").field("type", "category").endObject().endArray().endObject().endObject().endObject().endObject());
DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
Mapper fieldMapper = defaultMapper.mappers().getMapper("completion");
ParsedDocument parsedDocument = defaultMapper.parse(new SourceToParse("test", "type1", "1", BytesReference.bytes(jsonBuilder().startObject().startObject("completion").array("input", "suggestion5", "suggestion6", "suggestion7").startObject("contexts").array("ctx", "ctx1", true, 100).endObject().field("weight", 5).endObject().endObject()), XContentType.JSON));
IndexableField[] fields = parsedDocument.rootDoc().getFields(fieldMapper.name());
assertContextSuggestFields(fields, 3);
}
use of org.opensearch.index.mapper.Mapper in project OpenSearch by opensearch-project.
the class CategoryContextMappingTests method testIndexingWithNoContexts.
public void testIndexingWithNoContexts() throws Exception {
String mapping = Strings.toString(jsonBuilder().startObject().startObject("type1").startObject("properties").startObject("completion").field("type", "completion").startArray("contexts").startObject().field("name", "ctx").field("type", "category").endObject().endArray().endObject().endObject().endObject().endObject());
DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(mapping));
Mapper fieldMapper = defaultMapper.mappers().getMapper("completion");
ParsedDocument parsedDocument = defaultMapper.parse(new SourceToParse("test", "type1", "1", BytesReference.bytes(jsonBuilder().startObject().startArray("completion").startObject().array("input", "suggestion1", "suggestion2").field("weight", 3).endObject().startObject().array("input", "suggestion3", "suggestion4").field("weight", 4).endObject().startObject().array("input", "suggestion5", "suggestion6", "suggestion7").field("weight", 5).endObject().endArray().endObject()), XContentType.JSON));
IndexableField[] fields = parsedDocument.rootDoc().getFields(fieldMapper.name());
assertContextSuggestFields(fields, 7);
}
Aggregations