Examples with MapperService org.opensearch.index.mapper.MapperService used on opensource projects

Example 6 with MapperService

use of org.opensearch.index.mapper.MapperService in project OpenSearch by opensearch-project.

the class AnnotatedTextFieldMapperTests method testPositionIncrementGap.

public void testPositionIncrementGap() throws IOException {
    final int positionIncrementGap = randomIntBetween(1, 1000);
    MapperService mapperService = createMapperService(fieldMapping(b -> {
        b.field("type", "annotated_text");
        b.field("position_increment_gap", positionIncrementGap);
    }));
    ParsedDocument doc = mapperService.documentMapper().parse(source(b -> b.array("field", "a", "b")));
    IndexableField[] fields = doc.rootDoc().getFields("field");
    assertEquals(2, fields.length);
    assertEquals("a", fields[0].stringValue());
    assertEquals("b", fields[1].stringValue());
    withLuceneIndex(mapperService, iw -> iw.addDocument(doc.rootDoc()), reader -> {
        LeafReader leaf = reader.leaves().get(0).reader();
        TermsEnum terms = leaf.terms("field").iterator();
        assertTrue(terms.seekExact(new BytesRef("b")));
        PostingsEnum postings = terms.postings(null, PostingsEnum.POSITIONS);
        assertEquals(0, postings.nextDoc());
        assertEquals(positionIncrementGap + 1, postings.nextPosition());
    });
}

Example 7 with MapperService

use of org.opensearch.index.mapper.MapperService in project OpenSearch by opensearch-project.

the class RareClusterStateIT method testDelayedMappingPropagationOnReplica.

public void testDelayedMappingPropagationOnReplica() throws Exception {
    // This is essentially the same thing as testDelayedMappingPropagationOnPrimary
    // but for replicas
    // Here we want to test that everything goes well if the mappings that
    // are needed for a document are not available on the replica at the
    // time of indexing it
    final List<String> nodeNames = internalCluster().startNodes(2);
    assertFalse(client().admin().cluster().prepareHealth().setWaitForNodes("2").get().isTimedOut());
    final String master = internalCluster().getMasterName();
    assertThat(nodeNames, hasItem(master));
    String otherNode = null;
    for (String node : nodeNames) {
        if (node.equals(master) == false) {
            otherNode = node;
            break;
        }
    }
    assertNotNull(otherNode);
    // Force allocation of the primary on the master node by first only allocating on the master
    // and then allowing all nodes so that the replica gets allocated on the other node
    prepareCreate("index").setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1).put("index.routing.allocation.include._name", master)).get();
    client().admin().indices().prepareUpdateSettings("index").setSettings(Settings.builder().put("index.routing.allocation.include._name", "")).get();
    ensureGreen();
    // Check routing tables
    ClusterState state = client().admin().cluster().prepareState().get().getState();
    assertEquals(master, state.nodes().getMasterNode().getName());
    List<ShardRouting> shards = state.routingTable().allShards("index");
    assertThat(shards, hasSize(2));
    for (ShardRouting shard : shards) {
        if (shard.primary()) {
            // primary must be on the master
            assertEquals(state.nodes().getMasterNodeId(), shard.currentNodeId());
        } else {
            assertTrue(shard.active());
        }
    }
    // Block cluster state processing on the replica
    BlockClusterStateProcessing disruption = new BlockClusterStateProcessing(otherNode, random());
    internalCluster().setDisruptionScheme(disruption);
    disruption.startDisrupting();
    final ActionFuture<AcknowledgedResponse> putMappingResponse = executeAndCancelCommittedPublication(client().admin().indices().preparePutMapping("index").setSource("field", "type=long"));
    final Index index = resolveIndex("index");
    // Wait for mappings to be available on master
    assertBusy(() -> {
        final IndicesService indicesService = internalCluster().getInstance(IndicesService.class, master);
        final IndexService indexService = indicesService.indexServiceSafe(index);
        assertNotNull(indexService);
        final MapperService mapperService = indexService.mapperService();
        DocumentMapper mapper = mapperService.documentMapper(MapperService.SINGLE_MAPPING_NAME);
        assertNotNull(mapper);
        assertNotNull(mapper.mappers().getMapper("field"));
    });
    final ActionFuture<IndexResponse> docIndexResponse = client().prepareIndex("index").setId("1").setSource("field", 42).execute();
    assertBusy(() -> assertTrue(client().prepareGet("index", "1").get().isExists()));
    // index another document, this time using dynamic mappings.
    // The ack timeout of 0 on dynamic mapping updates makes it possible for the document to be indexed on the primary, even
    // if the dynamic mapping update is not applied on the replica yet.
    // this request does not change the cluster state, because the mapping is dynamic,
    // we need to await and cancel committed publication
    ActionFuture<IndexResponse> dynamicMappingsFut = executeAndCancelCommittedPublication(client().prepareIndex("index").setId("2").setSource("field2", 42));
    // ...and wait for second mapping to be available on master
    assertBusy(() -> {
        final IndicesService indicesService = internalCluster().getInstance(IndicesService.class, master);
        final IndexService indexService = indicesService.indexServiceSafe(index);
        assertNotNull(indexService);
        final MapperService mapperService = indexService.mapperService();
        DocumentMapper mapper = mapperService.documentMapper(MapperService.SINGLE_MAPPING_NAME);
        assertNotNull(mapper);
        assertNotNull(mapper.mappers().getMapper("field2"));
    });
    assertBusy(() -> assertTrue(client().prepareGet("index", "2").get().isExists()));
    // The mappings have not been propagated to the replica yet as a consequence the document count not be indexed
    // We wait on purpose to make sure that the document is not indexed because the shard operation is stalled
    // and not just because it takes time to replicate the indexing request to the replica
    Thread.sleep(100);
    assertFalse(putMappingResponse.isDone());
    assertFalse(docIndexResponse.isDone());
    // Now make sure the indexing request finishes successfully
    disruption.stopDisrupting();
    assertTrue(putMappingResponse.get(10, TimeUnit.SECONDS).isAcknowledged());
    assertThat(docIndexResponse.get(10, TimeUnit.SECONDS), instanceOf(IndexResponse.class));
    // both shards should have succeeded
    assertEquals(2, docIndexResponse.get(10, TimeUnit.SECONDS).getShardInfo().getTotal());
    assertThat(dynamicMappingsFut.get(10, TimeUnit.SECONDS).getResult(), equalTo(CREATED));
}

Example 8 with MapperService

use of org.opensearch.index.mapper.MapperService in project OpenSearch by opensearch-project.

the class PercolateQueryBuilder method doToQuery.

@Override
protected Query doToQuery(QueryShardContext context) throws IOException {
    if (context.allowExpensiveQueries() == false) {
        throw new OpenSearchException("[percolate] queries cannot be executed when '" + ALLOW_EXPENSIVE_QUERIES.getKey() + "' is set to false.");
    }
    // Call nowInMillis() so that this query becomes un-cacheable since we
    // can't be sure that it doesn't use now or scripts
    context.nowInMillis();
    if (indexedDocumentIndex != null || indexedDocumentId != null || documentSupplier != null) {
        throw new IllegalStateException("query builder must be rewritten first");
    }
    if (documents.isEmpty()) {
        throw new IllegalStateException("no document to percolate");
    }
    MappedFieldType fieldType = context.fieldMapper(field);
    if (fieldType == null) {
        throw new QueryShardException(context, "field [" + field + "] does not exist");
    }
    if (!(fieldType instanceof PercolatorFieldMapper.PercolatorFieldType)) {
        throw new QueryShardException(context, "expected field [" + field + "] to be of type [percolator], but is of type [" + fieldType.typeName() + "]");
    }
    final List<ParsedDocument> docs = new ArrayList<>();
    final DocumentMapper docMapper;
    final MapperService mapperService = context.getMapperService();
    String type = mapperService.documentMapper().type();
    if (documentType != null) {
        deprecationLogger.deprecate("percolate_with_document_type", DOCUMENT_TYPE_DEPRECATION_MESSAGE);
        if (documentType.equals(type) == false) {
            throw new IllegalArgumentException("specified document_type [" + documentType + "] is not equal to the actual type [" + type + "]");
        }
    }
    docMapper = mapperService.documentMapper(type);
    for (BytesReference document : documents) {
        docs.add(docMapper.parse(new SourceToParse(context.index().getName(), type, "_temp_id", document, documentXContentType)));
    }
    FieldNameAnalyzer fieldNameAnalyzer = (FieldNameAnalyzer) docMapper.mappers().indexAnalyzer();
    // Need to this custom impl because FieldNameAnalyzer is strict and the percolator sometimes isn't when
    // 'index.percolator.map_unmapped_fields_as_string' is enabled:
    Analyzer analyzer = new DelegatingAnalyzerWrapper(Analyzer.PER_FIELD_REUSE_STRATEGY) {

        @Override
        protected Analyzer getWrappedAnalyzer(String fieldName) {
            Analyzer analyzer = fieldNameAnalyzer.analyzers().get(fieldName);
            if (analyzer != null) {
                return analyzer;
            } else {
                return context.getIndexAnalyzers().getDefaultIndexAnalyzer();
            }
        }
    };
    final IndexSearcher docSearcher;
    final boolean excludeNestedDocuments;
    if (docs.size() > 1 || docs.get(0).docs().size() > 1) {
        assert docs.size() != 1 || docMapper.hasNestedObjects();
        docSearcher = createMultiDocumentSearcher(analyzer, docs);
        excludeNestedDocuments = docMapper.hasNestedObjects() && docs.stream().map(ParsedDocument::docs).mapToInt(List::size).anyMatch(size -> size > 1);
    } else {
        MemoryIndex memoryIndex = MemoryIndex.fromDocument(docs.get(0).rootDoc(), analyzer, true, false);
        docSearcher = memoryIndex.createSearcher();
        docSearcher.setQueryCache(null);
        excludeNestedDocuments = false;
    }
    PercolatorFieldMapper.PercolatorFieldType pft = (PercolatorFieldMapper.PercolatorFieldType) fieldType;
    String name = this.name != null ? this.name : pft.name();
    QueryShardContext percolateShardContext = wrap(context);
    PercolatorFieldMapper.configureContext(percolateShardContext, pft.mapUnmappedFieldsAsText);
    ;
    PercolateQuery.QueryStore queryStore = createStore(pft.queryBuilderField, percolateShardContext);
    return pft.percolateQuery(name, queryStore, documents, docSearcher, excludeNestedDocuments, context.indexVersionCreated());
}

Example 9 with MapperService

use of org.opensearch.index.mapper.MapperService in project OpenSearch by opensearch-project.

the class UpdateMappingIntegrationIT method assertConcreteMappingsOnAll.

/**
 * Waits until mappings for the provided fields exist on all nodes. Note, this waits for the current
 * started shards and checks for concrete mappings.
 */
private void assertConcreteMappingsOnAll(final String index, final String... fieldNames) {
    Set<String> nodes = internalCluster().nodesInclude(index);
    assertThat(nodes, Matchers.not(Matchers.emptyIterable()));
    for (String node : nodes) {
        IndicesService indicesService = internalCluster().getInstance(IndicesService.class, node);
        IndexService indexService = indicesService.indexService(resolveIndex(index));
        assertThat("index service doesn't exists on " + node, indexService, notNullValue());
        MapperService mapperService = indexService.mapperService();
        for (String fieldName : fieldNames) {
            MappedFieldType fieldType = mapperService.fieldType(fieldName);
            assertNotNull("field " + fieldName + " doesn't exists on " + node, fieldType);
        }
    }
    assertMappingOnMaster(index, fieldNames);
}

Example 10 with MapperService

use of org.opensearch.index.mapper.MapperService in project OpenSearch by opensearch-project.

the class TransportFieldCapabilitiesIndexAction method shardOperation.

private FieldCapabilitiesIndexResponse shardOperation(final FieldCapabilitiesIndexRequest request) throws IOException {
    if (canMatchShard(request) == false) {
        return new FieldCapabilitiesIndexResponse(request.index(), Collections.emptyMap(), false);
    }
    ShardId shardId = request.shardId();
    MapperService mapperService = indicesService.indexServiceSafe(shardId.getIndex()).mapperService();
    Set<String> fieldNames = new HashSet<>();
    for (String field : request.fields()) {
        fieldNames.addAll(mapperService.simpleMatchToFullName(field));
    }
    Predicate<String> fieldPredicate = indicesService.getFieldFilter().apply(shardId.getIndexName());
    Map<String, IndexFieldCapabilities> responseMap = new HashMap<>();
    for (String field : fieldNames) {
        MappedFieldType ft = mapperService.fieldType(field);
        if (ft != null) {
            if (indicesService.isMetadataField(mapperService.getIndexSettings().getIndexVersionCreated(), field) || fieldPredicate.test(ft.name())) {
                IndexFieldCapabilities fieldCap = new IndexFieldCapabilities(field, ft.familyTypeName(), ft.isSearchable(), ft.isAggregatable(), ft.meta());
                responseMap.put(field, fieldCap);
            } else {
                continue;
            }
            // add nested and object fields
            int dotIndex = ft.name().lastIndexOf('.');
            while (dotIndex > -1) {
                String parentField = ft.name().substring(0, dotIndex);
                if (responseMap.containsKey(parentField)) {
                    // we added this path on another field already
                    break;
                }
                // checks if the parent field contains sub-fields
                if (mapperService.fieldType(parentField) == null) {
                    // no field type, it must be an object field
                    ObjectMapper mapper = mapperService.getObjectMapper(parentField);
                    String type = mapper.nested().isNested() ? "nested" : "object";
                    IndexFieldCapabilities fieldCap = new IndexFieldCapabilities(parentField, type, false, false, Collections.emptyMap());
                    responseMap.put(parentField, fieldCap);
                }
                dotIndex = parentField.lastIndexOf('.');
            }
        }
    }
    return new FieldCapabilitiesIndexResponse(request.index(), responseMap, true);
}