Examples with MapperRegistry org.opensearch.indices.mapper.MapperRegistry used on opensource projects

Example 1 with MapperRegistry

use of org.opensearch.indices.mapper.MapperRegistry in project OpenSearch by opensearch-project.

the class IndicesModuleTests method testGetFieldFilter.

public void testGetFieldFilter() {
    List<MapperPlugin> mapperPlugins = Arrays.asList(new MapperPlugin() {

        @Override
        public Function<String, Predicate<String>> getFieldFilter() {
            return MapperPlugin.NOOP_FIELD_FILTER;
        }
    }, new MapperPlugin() {

        @Override
        public Function<String, Predicate<String>> getFieldFilter() {
            return index -> index.equals("hidden_index") ? field -> false : MapperPlugin.NOOP_FIELD_PREDICATE;
        }
    }, new MapperPlugin() {

        @Override
        public Function<String, Predicate<String>> getFieldFilter() {
            return index -> field -> field.equals("hidden_field") == false;
        }
    }, new MapperPlugin() {

        @Override
        public Function<String, Predicate<String>> getFieldFilter() {
            return index -> index.equals("filtered") ? field -> field.equals("visible") : MapperPlugin.NOOP_FIELD_PREDICATE;
        }
    });
    IndicesModule indicesModule = new IndicesModule(mapperPlugins);
    MapperRegistry mapperRegistry = indicesModule.getMapperRegistry();
    Function<String, Predicate<String>> fieldFilter = mapperRegistry.getFieldFilter();
    assertNotSame(MapperPlugin.NOOP_FIELD_FILTER, fieldFilter);
    assertFalse(fieldFilter.apply("hidden_index").test(randomAlphaOfLengthBetween(3, 5)));
    assertTrue(fieldFilter.apply(randomAlphaOfLengthBetween(3, 5)).test(randomAlphaOfLengthBetween(3, 5)));
    assertFalse(fieldFilter.apply(randomAlphaOfLengthBetween(3, 5)).test("hidden_field"));
    assertFalse(fieldFilter.apply("filtered").test(randomAlphaOfLengthBetween(3, 5)));
    assertFalse(fieldFilter.apply("filtered").test("hidden_field"));
    assertTrue(fieldFilter.apply("filtered").test("visible"));
    assertFalse(fieldFilter.apply("hidden_index").test("visible"));
    assertTrue(fieldFilter.apply(randomAlphaOfLengthBetween(3, 5)).test("visible"));
    assertFalse(fieldFilter.apply("hidden_index").test("hidden_field"));
}

Example 2 with MapperRegistry

use of org.opensearch.indices.mapper.MapperRegistry in project OpenSearch by opensearch-project.

the class IndicesModuleTests method testBuiltinWithPlugins.

public void testBuiltinWithPlugins() {
    IndicesModule noPluginsModule = new IndicesModule(Collections.emptyList());
    IndicesModule module = new IndicesModule(fakePlugins);
    MapperRegistry registry = module.getMapperRegistry();
    assertThat(registry.getMapperParsers().size(), greaterThan(noPluginsModule.getMapperRegistry().getMapperParsers().size()));
    assertThat(registry.getMetadataMapperParsers(Version.CURRENT).size(), greaterThan(noPluginsModule.getMapperRegistry().getMetadataMapperParsers(Version.CURRENT).size()));
    Map<String, MetadataFieldMapper.TypeParser> metadataMapperParsers = module.getMapperRegistry().getMetadataMapperParsers(Version.CURRENT);
    Iterator<String> iterator = metadataMapperParsers.keySet().iterator();
    assertEquals(IgnoredFieldMapper.NAME, iterator.next());
    String last = null;
    while (iterator.hasNext()) {
        last = iterator.next();
    }
    assertEquals(FieldNamesFieldMapper.NAME, last);
}

Example 3 with MapperRegistry

use of org.opensearch.indices.mapper.MapperRegistry in project OpenSearch by opensearch-project.

the class AggregatorTestCase method testSupportedFieldTypes.

/**
 * This test will validate that an aggregator succeeds or fails to run against all the field types
 * that are registered in {@link IndicesModule} (e.g. all the core field types).  An aggregator
 * is provided by the implementor class, and it is executed against each field type in turn.  If
 * an exception is thrown when the field is supported, that will fail the test.  Similarly, if
 * an exception _is not_ thrown when a field is unsupported, that will also fail the test.
 *
 * Exception types/messages are not currently checked, just presence/absence of an exception.
 */
public void testSupportedFieldTypes() throws IOException {
    MapperRegistry mapperRegistry = new IndicesModule(Collections.emptyList()).getMapperRegistry();
    Settings settings = Settings.builder().put("index.version.created", Version.CURRENT.id).build();
    String fieldName = "typeTestFieldName";
    List<ValuesSourceType> supportedVSTypes = getSupportedValuesSourceTypes();
    List<String> unsupportedMappedFieldTypes = unsupportedMappedFieldTypes();
    if (supportedVSTypes.isEmpty()) {
        // If the test says it doesn't support any VStypes, it has not been converted yet so skip
        return;
    }
    for (Map.Entry<String, Mapper.TypeParser> mappedType : mapperRegistry.getMapperParsers().entrySet()) {
        // Some field types should not be tested, or require more work and are not ready yet
        if (TYPE_TEST_DENYLIST.contains(mappedType.getKey())) {
            continue;
        }
        Map<String, Object> source = new HashMap<>();
        source.put("type", mappedType.getKey());
        // Text is the only field that doesn't support DVs, instead FD
        if (mappedType.getKey().equals(TextFieldMapper.CONTENT_TYPE) == false) {
            source.put("doc_values", "true");
        }
        Mapper.Builder builder = mappedType.getValue().parse(fieldName, source, new MockParserContext());
        FieldMapper mapper = (FieldMapper) builder.build(new BuilderContext(settings, new ContentPath()));
        MappedFieldType fieldType = mapper.fieldType();
        // Non-aggregatable fields are not testable (they will throw an error on all aggs anyway), so skip
        if (fieldType.isAggregatable() == false) {
            continue;
        }
        try (Directory directory = newDirectory()) {
            RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory);
            writeTestDoc(fieldType, fieldName, indexWriter);
            indexWriter.close();
            try (IndexReader indexReader = DirectoryReader.open(directory)) {
                IndexSearcher indexSearcher = newIndexSearcher(indexReader);
                AggregationBuilder aggregationBuilder = createAggBuilderForTypeTest(fieldType, fieldName);
                ValuesSourceType vst = fieldToVST(fieldType);
                // TODO in the future we can make this more explicit with expectThrows(), when the exceptions are standardized
                AssertionError failure = null;
                try {
                    searchAndReduce(indexSearcher, new MatchAllDocsQuery(), aggregationBuilder, fieldType);
                    if (supportedVSTypes.contains(vst) == false || unsupportedMappedFieldTypes.contains(fieldType.typeName())) {
                        failure = new AssertionError("Aggregator [" + aggregationBuilder.getType() + "] should not support field type [" + fieldType.typeName() + "] but executing against the field did not throw an exception");
                    }
                } catch (Exception | AssertionError e) {
                    if (supportedVSTypes.contains(vst) && unsupportedMappedFieldTypes.contains(fieldType.typeName()) == false) {
                        failure = new AssertionError("Aggregator [" + aggregationBuilder.getType() + "] supports field type [" + fieldType.typeName() + "] but executing against the field threw an exception: [" + e.getMessage() + "]", e);
                    }
                }
                if (failure != null) {
                    throw failure;
                }
            }
        }
    }
}

Example 4 with MapperRegistry

use of org.opensearch.indices.mapper.MapperRegistry in project OpenSearch by opensearch-project.

the class MetadataIndexUpgradeService method checkMappingsCompatibility.

/**
 * Checks the mappings for compatibility with the current version
 */
private void checkMappingsCompatibility(IndexMetadata indexMetadata) {
    try {
        // We cannot instantiate real analysis server or similarity service at this point because the node
        // might not have been started yet. However, we don't really need real analyzers or similarities at
        // this stage - so we can fake it using constant maps accepting every key.
        // This is ok because all used similarities and analyzers for this index were known before the upgrade.
        // Missing analyzers and similarities plugin will still trigger the appropriate error during the
        // actual upgrade.
        IndexSettings indexSettings = new IndexSettings(indexMetadata, this.settings);
        final Map<String, TriFunction<Settings, Version, ScriptService, Similarity>> similarityMap = new AbstractMap<String, TriFunction<Settings, Version, ScriptService, Similarity>>() {

            @Override
            public boolean containsKey(Object key) {
                return true;
            }

            @Override
            public TriFunction<Settings, Version, ScriptService, Similarity> get(Object key) {
                assert key instanceof String : "key must be a string but was: " + key.getClass();
                return (settings, version, scriptService) -> new BM25Similarity();
            }

            // this entrySet impl isn't fully correct but necessary as SimilarityService will iterate
            // over all similarities
            @Override
            public Set<Entry<String, TriFunction<Settings, Version, ScriptService, Similarity>>> entrySet() {
                return Collections.emptySet();
            }
        };
        SimilarityService similarityService = new SimilarityService(indexSettings, null, similarityMap);
        final NamedAnalyzer fakeDefault = new NamedAnalyzer("default", AnalyzerScope.INDEX, new Analyzer() {

            @Override
            protected TokenStreamComponents createComponents(String fieldName) {
                throw new UnsupportedOperationException("shouldn't be here");
            }
        });
        final Map<String, NamedAnalyzer> analyzerMap = new AbstractMap<String, NamedAnalyzer>() {

            @Override
            public NamedAnalyzer get(Object key) {
                assert key instanceof String : "key must be a string but was: " + key.getClass();
                return new NamedAnalyzer((String) key, AnalyzerScope.INDEX, fakeDefault.analyzer());
            }

            // this entrySet impl isn't fully correct but necessary as IndexAnalyzers will iterate
            // over all analyzers to close them
            @Override
            public Set<Entry<String, NamedAnalyzer>> entrySet() {
                return Collections.emptySet();
            }
        };
        try (IndexAnalyzers fakeIndexAnalzyers = new IndexAnalyzers(analyzerMap, analyzerMap, analyzerMap)) {
            MapperService mapperService = new MapperService(indexSettings, fakeIndexAnalzyers, xContentRegistry, similarityService, mapperRegistry, () -> null, () -> false, scriptService);
            mapperService.merge(indexMetadata, MapperService.MergeReason.MAPPING_RECOVERY);
        }
    } catch (Exception ex) {
        // Wrap the inner exception so we have the index name in the exception message
        throw new IllegalStateException("unable to upgrade the mappings for the index [" + indexMetadata.getIndex() + "]", ex);
    }
}

Example 5 with MapperRegistry

use of org.opensearch.indices.mapper.MapperRegistry in project OpenSearch by opensearch-project.

the class MapperServiceTestCase method createMapperService.

/**
 * Create a {@link MapperService} like we would for an index.
 */
protected final MapperService createMapperService(Version version, XContentBuilder mapping) throws IOException {
    IndexMetadata meta = IndexMetadata.builder("index").settings(Settings.builder().put("index.version.created", version)).numberOfReplicas(0).numberOfShards(1).build();
    IndexSettings indexSettings = new IndexSettings(meta, getIndexSettings());
    MapperRegistry mapperRegistry = new IndicesModule(getPlugins().stream().filter(p -> p instanceof MapperPlugin).map(p -> (MapperPlugin) p).collect(toList())).getMapperRegistry();
    ScriptModule scriptModule = new ScriptModule(Settings.EMPTY, getPlugins().stream().filter(p -> p instanceof ScriptPlugin).map(p -> (ScriptPlugin) p).collect(toList()));
    ScriptService scriptService = new ScriptService(getIndexSettings(), scriptModule.engines, scriptModule.contexts);
    SimilarityService similarityService = new SimilarityService(indexSettings, scriptService, emptyMap());
    MapperService mapperService = new MapperService(indexSettings, createIndexAnalyzers(indexSettings), xContentRegistry(), similarityService, mapperRegistry, () -> {
        throw new UnsupportedOperationException();
    }, () -> true, scriptService);
    merge(mapperService, mapping);
    return mapperService;
}