Examples with ParsedDocument org.opensearch.index.mapper.ParsedDocument used on opensource projects

Example 96 with ParsedDocument

use of org.opensearch.index.mapper.ParsedDocument in project OpenSearch by opensearch-project.

the class InternalEngine method deleteInLucene.

private DeleteResult deleteInLucene(Delete delete, DeletionStrategy plan) throws IOException {
    assert assertMaxSeqNoOfUpdatesIsAdvanced(delete.uid(), delete.seqNo(), false, false);
    try {
        final ParsedDocument tombstone = engineConfig.getTombstoneDocSupplier().newDeleteTombstoneDoc(delete.type(), delete.id());
        assert tombstone.docs().size() == 1 : "Tombstone doc should have single doc [" + tombstone + "]";
        tombstone.updateSeqID(delete.seqNo(), delete.primaryTerm());
        tombstone.version().setLongValue(plan.versionOfDeletion);
        final ParseContext.Document doc = tombstone.docs().get(0);
        assert doc.getField(SeqNoFieldMapper.TOMBSTONE_NAME) != null : "Delete tombstone document but _tombstone field is not set [" + doc + " ]";
        doc.add(softDeletesField);
        if (plan.addStaleOpToLucene || plan.currentlyDeleted) {
            indexWriter.addDocument(doc);
        } else {
            indexWriter.softUpdateDocument(delete.uid(), doc, softDeletesField);
        }
        return new DeleteResult(plan.versionOfDeletion, delete.primaryTerm(), delete.seqNo(), plan.currentlyDeleted == false);
    } catch (final Exception ex) {
        /*
             * Document level failures when deleting are unexpected, we likely hit something fatal such as the Lucene index being corrupt,
             * or the Lucene document limit. We have already issued a sequence number here so this is fatal, fail the engine.
             */
        if (ex instanceof AlreadyClosedException == false && indexWriter.getTragicException() == null) {
            final String reason = String.format(Locale.ROOT, "delete id[%s] origin [%s] seq#[%d] failed at the document level", delete.id(), delete.origin(), delete.seqNo());
            failEngine(reason, ex);
        }
        throw ex;
    }
}

Example 97 with ParsedDocument

use of org.opensearch.index.mapper.ParsedDocument in project OpenSearch by opensearch-project.

the class TermVectorsService method generateTermVectorsFromDoc.

private static Fields generateTermVectorsFromDoc(IndexShard indexShard, TermVectorsRequest request) throws IOException {
    // parse the document, at the moment we do update the mapping, just like percolate
    ParsedDocument parsedDocument = parseDocument(indexShard, indexShard.shardId().getIndexName(), request.doc(), request.xContentType(), request.routing());
    // select the right fields and generate term vectors
    ParseContext.Document doc = parsedDocument.rootDoc();
    Set<String> seenFields = new HashSet<>();
    Collection<DocumentField> documentFields = new HashSet<>();
    for (IndexableField field : doc.getFields()) {
        MappedFieldType fieldType = indexShard.mapperService().fieldType(field.name());
        if (!isValidField(fieldType)) {
            continue;
        }
        if (request.selectedFields() != null && !request.selectedFields().contains(field.name())) {
            continue;
        }
        if (seenFields.contains(field.name())) {
            continue;
        } else {
            seenFields.add(field.name());
        }
        String[] values = getValues(doc.getFields(field.name()));
        documentFields.add(new DocumentField(field.name(), Arrays.asList((Object[]) values)));
    }
    return generateTermVectors(indexShard, XContentHelper.convertToMap(parsedDocument.source(), true, request.xContentType()).v2(), documentFields, request.offsets(), request.perFieldAnalyzer(), seenFields);
}

Example 98 with ParsedDocument

use of org.opensearch.index.mapper.ParsedDocument in project OpenSearch by opensearch-project.

the class TermVectorsService method parseDocument.

private static ParsedDocument parseDocument(IndexShard indexShard, String index, BytesReference doc, XContentType xContentType, String routing) {
    MapperService mapperService = indexShard.mapperService();
    DocumentMapperForType docMapper = mapperService.documentMapperWithAutoCreate(MapperService.SINGLE_MAPPING_NAME);
    ParsedDocument parsedDocument = docMapper.getDocumentMapper().parse(new SourceToParse(index, MapperService.SINGLE_MAPPING_NAME, "_id_for_tv_api", doc, xContentType, routing));
    if (docMapper.getMapping() != null) {
        parsedDocument.addDynamicMappingsUpdate(docMapper.getMapping());
    }
    return parsedDocument;
}

Example 99 with ParsedDocument

use of org.opensearch.index.mapper.ParsedDocument in project OpenSearch by opensearch-project.

the class ShardGetService method innerGetLoadFromStoredFields.

private GetResult innerGetLoadFromStoredFields(String id, String[] storedFields, FetchSourceContext fetchSourceContext, Engine.GetResult get, MapperService mapperService) {
    assert get.exists() : "method should only be called if document could be retrieved";
    // check first if stored fields to be loaded don't contain an object field
    DocumentMapper docMapper = mapperService.documentMapper();
    if (storedFields != null) {
        for (String field : storedFields) {
            Mapper fieldMapper = docMapper.mappers().getMapper(field);
            if (fieldMapper == null) {
                if (docMapper.objectMappers().get(field) != null) {
                    // Only fail if we know it is a object field, missing paths / fields shouldn't fail.
                    throw new IllegalArgumentException("field [" + field + "] isn't a leaf field");
                }
            }
        }
    }
    Map<String, DocumentField> documentFields = null;
    Map<String, DocumentField> metadataFields = null;
    BytesReference source = null;
    DocIdAndVersion docIdAndVersion = get.docIdAndVersion();
    // force fetching source if we read from translog and need to recreate stored fields
    boolean forceSourceForComputingTranslogStoredFields = get.isFromTranslog() && storedFields != null && Stream.of(storedFields).anyMatch(f -> TranslogLeafReader.ALL_FIELD_NAMES.contains(f) == false);
    FieldsVisitor fieldVisitor = buildFieldsVisitors(storedFields, forceSourceForComputingTranslogStoredFields ? FetchSourceContext.FETCH_SOURCE : fetchSourceContext);
    if (fieldVisitor != null) {
        try {
            docIdAndVersion.reader.document(docIdAndVersion.docId, fieldVisitor);
        } catch (IOException e) {
            throw new OpenSearchException("Failed to get id [" + id + "]", e);
        }
        source = fieldVisitor.source();
        // in case we read from translog, some extra steps are needed to make _source consistent and to load stored fields
        if (get.isFromTranslog()) {
            // just make source consistent by reapplying source filters from mapping (possibly also nulling the source)
            if (forceSourceForComputingTranslogStoredFields == false) {
                try {
                    source = indexShard.mapperService().documentMapper().sourceMapper().applyFilters(source, null);
                } catch (IOException e) {
                    throw new OpenSearchException("Failed to reapply filters for [" + id + "] after reading from translog", e);
                }
            } else {
                // Slow path: recreate stored fields from original source
                assert source != null : "original source in translog must exist";
                SourceToParse sourceToParse = new SourceToParse(shardId.getIndexName(), MapperService.SINGLE_MAPPING_NAME, id, source, XContentHelper.xContentType(source), fieldVisitor.routing());
                ParsedDocument doc = indexShard.mapperService().documentMapper().parse(sourceToParse);
                assert doc.dynamicMappingsUpdate() == null : "mapping updates should not be required on already-indexed doc";
                // update special fields
                doc.updateSeqID(docIdAndVersion.seqNo, docIdAndVersion.primaryTerm);
                doc.version().setLongValue(docIdAndVersion.version);
                // retrieve stored fields from parsed doc
                fieldVisitor = buildFieldsVisitors(storedFields, fetchSourceContext);
                for (IndexableField indexableField : doc.rootDoc().getFields()) {
                    IndexableFieldType fieldType = indexableField.fieldType();
                    if (fieldType.stored()) {
                        FieldInfo fieldInfo = new FieldInfo(indexableField.name(), 0, false, false, false, IndexOptions.NONE, DocValuesType.NONE, -1, Collections.emptyMap(), 0, 0, 0, false);
                        StoredFieldVisitor.Status status = fieldVisitor.needsField(fieldInfo);
                        if (status == StoredFieldVisitor.Status.YES) {
                            if (indexableField.numericValue() != null) {
                                fieldVisitor.objectField(fieldInfo, indexableField.numericValue());
                            } else if (indexableField.binaryValue() != null) {
                                fieldVisitor.binaryField(fieldInfo, indexableField.binaryValue());
                            } else if (indexableField.stringValue() != null) {
                                fieldVisitor.objectField(fieldInfo, indexableField.stringValue());
                            }
                        } else if (status == StoredFieldVisitor.Status.STOP) {
                            break;
                        }
                    }
                }
                // retrieve source (with possible transformations, e.g. source filters
                source = fieldVisitor.source();
            }
        }
        // put stored fields into result objects
        if (!fieldVisitor.fields().isEmpty()) {
            fieldVisitor.postProcess(mapperService);
            documentFields = new HashMap<>();
            metadataFields = new HashMap<>();
            for (Map.Entry<String, List<Object>> entry : fieldVisitor.fields().entrySet()) {
                if (mapperService.isMetadataField(entry.getKey())) {
                    metadataFields.put(entry.getKey(), new DocumentField(entry.getKey(), entry.getValue()));
                } else {
                    documentFields.put(entry.getKey(), new DocumentField(entry.getKey(), entry.getValue()));
                }
            }
        }
    }
    if (source != null) {
        // apply request-level source filtering
        if (fetchSourceContext.fetchSource() == false) {
            source = null;
        } else if (fetchSourceContext.includes().length > 0 || fetchSourceContext.excludes().length > 0) {
            Map<String, Object> sourceAsMap;
            // TODO: The source might be parsed and available in the sourceLookup but that one uses unordered maps so different.
            // Do we care?
            Tuple<XContentType, Map<String, Object>> typeMapTuple = XContentHelper.convertToMap(source, true);
            XContentType sourceContentType = typeMapTuple.v1();
            sourceAsMap = typeMapTuple.v2();
            sourceAsMap = XContentMapValues.filter(sourceAsMap, fetchSourceContext.includes(), fetchSourceContext.excludes());
            try {
                source = BytesReference.bytes(XContentFactory.contentBuilder(sourceContentType).map(sourceAsMap));
            } catch (IOException e) {
                throw new OpenSearchException("Failed to get id [" + id + "] with includes/excludes set", e);
            }
        }
    }
    if (!fetchSourceContext.fetchSource()) {
        source = null;
    }
    if (source != null && get.isFromTranslog()) {
        // reapply source filters from mapping (possibly also nulling the source)
        try {
            source = docMapper.sourceMapper().applyFilters(source, null);
        } catch (IOException e) {
            throw new OpenSearchException("Failed to reapply filters for [" + id + "] after reading from translog", e);
        }
    }
    if (source != null && (fetchSourceContext.includes().length > 0 || fetchSourceContext.excludes().length > 0)) {
        Map<String, Object> sourceAsMap;
        // TODO: The source might parsed and available in the sourceLookup but that one uses unordered maps so different. Do we care?
        Tuple<XContentType, Map<String, Object>> typeMapTuple = XContentHelper.convertToMap(source, true);
        XContentType sourceContentType = typeMapTuple.v1();
        sourceAsMap = typeMapTuple.v2();
        sourceAsMap = XContentMapValues.filter(sourceAsMap, fetchSourceContext.includes(), fetchSourceContext.excludes());
        try {
            source = BytesReference.bytes(XContentFactory.contentBuilder(sourceContentType).map(sourceAsMap));
        } catch (IOException e) {
            throw new OpenSearchException("Failed to get id [" + id + "] with includes/excludes set", e);
        }
    }
    return new GetResult(shardId.getIndexName(), id, get.docIdAndVersion().seqNo, get.docIdAndVersion().primaryTerm, get.version(), get.exists(), source, documentFields, metadataFields);
}

Example 100 with ParsedDocument

use of org.opensearch.index.mapper.ParsedDocument in project OpenSearch by opensearch-project.

the class IndexShard method prepareIndex.

public static Engine.Index prepareIndex(DocumentMapperForType docMapper, SourceToParse source, long seqNo, long primaryTerm, long version, VersionType versionType, Engine.Operation.Origin origin, long autoGeneratedIdTimestamp, boolean isRetry, long ifSeqNo, long ifPrimaryTerm) {
    long startTime = System.nanoTime();
    ParsedDocument doc = docMapper.getDocumentMapper().parse(source);
    if (docMapper.getMapping() != null) {
        doc.addDynamicMappingsUpdate(docMapper.getMapping());
    }
    Term uid = new Term(IdFieldMapper.NAME, Uid.encodeId(doc.id()));
    return new Engine.Index(uid, doc, seqNo, primaryTerm, version, versionType, origin, startTime, autoGeneratedIdTimestamp, isRetry, ifSeqNo, ifPrimaryTerm);
}