Examples with SimilarityService org.opensearch.index.similarity.SimilarityService used on opensource projects

Example 1 with SimilarityService

use of org.opensearch.index.similarity.SimilarityService in project OpenSearch by opensearch-project.

the class IndexModuleTests method testAddSimilarity.

public void testAddSimilarity() throws IOException {
    Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).put("index.similarity.my_similarity.type", "test_similarity").put("index.similarity.my_similarity.key", "there is a key").put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()).build();
    final IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("foo", settings);
    IndexModule module = createIndexModule(indexSettings, emptyAnalysisRegistry);
    module.addSimilarity("test_similarity", (providerSettings, indexCreatedVersion, scriptService) -> new TestSimilarity(providerSettings.get("key")));
    IndexService indexService = newIndexService(module);
    SimilarityService similarityService = indexService.similarityService();
    Similarity similarity = similarityService.getSimilarity("my_similarity").get();
    assertNotNull(similarity);
    assertThat(similarity, Matchers.instanceOf(NonNegativeScoresSimilarity.class));
    similarity = ((NonNegativeScoresSimilarity) similarity).getDelegate();
    assertThat(similarity, Matchers.instanceOf(TestSimilarity.class));
    assertEquals("my_similarity", similarityService.getSimilarity("my_similarity").name());
    assertEquals("there is a key", ((TestSimilarity) similarity).key);
    indexService.close("simon says", false);
}

Example 2 with SimilarityService

use of org.opensearch.index.similarity.SimilarityService in project OpenSearch by opensearch-project.

the class MetadataIndexUpgradeService method checkMappingsCompatibility.

/**
 * Checks the mappings for compatibility with the current version
 */
private void checkMappingsCompatibility(IndexMetadata indexMetadata) {
    try {
        // We cannot instantiate real analysis server or similarity service at this point because the node
        // might not have been started yet. However, we don't really need real analyzers or similarities at
        // this stage - so we can fake it using constant maps accepting every key.
        // This is ok because all used similarities and analyzers for this index were known before the upgrade.
        // Missing analyzers and similarities plugin will still trigger the appropriate error during the
        // actual upgrade.
        IndexSettings indexSettings = new IndexSettings(indexMetadata, this.settings);
        final Map<String, TriFunction<Settings, Version, ScriptService, Similarity>> similarityMap = new AbstractMap<String, TriFunction<Settings, Version, ScriptService, Similarity>>() {

            @Override
            public boolean containsKey(Object key) {
                return true;
            }

            @Override
            public TriFunction<Settings, Version, ScriptService, Similarity> get(Object key) {
                assert key instanceof String : "key must be a string but was: " + key.getClass();
                return (settings, version, scriptService) -> new BM25Similarity();
            }

            // this entrySet impl isn't fully correct but necessary as SimilarityService will iterate
            // over all similarities
            @Override
            public Set<Entry<String, TriFunction<Settings, Version, ScriptService, Similarity>>> entrySet() {
                return Collections.emptySet();
            }
        };
        SimilarityService similarityService = new SimilarityService(indexSettings, null, similarityMap);
        final NamedAnalyzer fakeDefault = new NamedAnalyzer("default", AnalyzerScope.INDEX, new Analyzer() {

            @Override
            protected TokenStreamComponents createComponents(String fieldName) {
                throw new UnsupportedOperationException("shouldn't be here");
            }
        });
        final Map<String, NamedAnalyzer> analyzerMap = new AbstractMap<String, NamedAnalyzer>() {

            @Override
            public NamedAnalyzer get(Object key) {
                assert key instanceof String : "key must be a string but was: " + key.getClass();
                return new NamedAnalyzer((String) key, AnalyzerScope.INDEX, fakeDefault.analyzer());
            }

            // this entrySet impl isn't fully correct but necessary as IndexAnalyzers will iterate
            // over all analyzers to close them
            @Override
            public Set<Entry<String, NamedAnalyzer>> entrySet() {
                return Collections.emptySet();
            }
        };
        try (IndexAnalyzers fakeIndexAnalzyers = new IndexAnalyzers(analyzerMap, analyzerMap, analyzerMap)) {
            MapperService mapperService = new MapperService(indexSettings, fakeIndexAnalzyers, xContentRegistry, similarityService, mapperRegistry, () -> null, () -> false, scriptService);
            mapperService.merge(indexMetadata, MapperService.MergeReason.MAPPING_RECOVERY);
        }
    } catch (Exception ex) {
        // Wrap the inner exception so we have the index name in the exception message
        throw new IllegalStateException("unable to upgrade the mappings for the index [" + indexMetadata.getIndex() + "]", ex);
    }
}

Example 3 with SimilarityService

use of org.opensearch.index.similarity.SimilarityService in project OpenSearch by opensearch-project.

the class IndexModule method newIndexService.

public IndexService newIndexService(IndexService.IndexCreationContext indexCreationContext, NodeEnvironment environment, NamedXContentRegistry xContentRegistry, IndexService.ShardStoreDeleter shardStoreDeleter, CircuitBreakerService circuitBreakerService, BigArrays bigArrays, ThreadPool threadPool, ScriptService scriptService, ClusterService clusterService, Client client, IndicesQueryCache indicesQueryCache, MapperRegistry mapperRegistry, IndicesFieldDataCache indicesFieldDataCache, NamedWriteableRegistry namedWriteableRegistry, BooleanSupplier idFieldDataEnabled, ValuesSourceRegistry valuesSourceRegistry) throws IOException {
    final IndexEventListener eventListener = freeze();
    Function<IndexService, CheckedFunction<DirectoryReader, DirectoryReader, IOException>> readerWrapperFactory = indexReaderWrapper.get() == null ? (shard) -> null : indexReaderWrapper.get();
    eventListener.beforeIndexCreated(indexSettings.getIndex(), indexSettings.getSettings());
    final IndexStorePlugin.DirectoryFactory directoryFactory = getDirectoryFactory(indexSettings, directoryFactories);
    final IndexStorePlugin.RecoveryStateFactory recoveryStateFactory = getRecoveryStateFactory(indexSettings, recoveryStateFactories);
    QueryCache queryCache = null;
    IndexAnalyzers indexAnalyzers = null;
    boolean success = false;
    try {
        if (indexSettings.getValue(INDEX_QUERY_CACHE_ENABLED_SETTING)) {
            BiFunction<IndexSettings, IndicesQueryCache, QueryCache> queryCacheProvider = forceQueryCacheProvider.get();
            if (queryCacheProvider == null) {
                queryCache = new IndexQueryCache(indexSettings, indicesQueryCache);
            } else {
                queryCache = queryCacheProvider.apply(indexSettings, indicesQueryCache);
            }
        } else {
            queryCache = new DisabledQueryCache(indexSettings);
        }
        if (IndexService.needsMapperService(indexSettings, indexCreationContext)) {
            indexAnalyzers = analysisRegistry.build(indexSettings);
        }
        final IndexService indexService = new IndexService(indexSettings, indexCreationContext, environment, xContentRegistry, new SimilarityService(indexSettings, scriptService, similarities), shardStoreDeleter, indexAnalyzers, engineFactory, engineConfigFactory, circuitBreakerService, bigArrays, threadPool, scriptService, clusterService, client, queryCache, directoryFactory, eventListener, readerWrapperFactory, mapperRegistry, indicesFieldDataCache, searchOperationListeners, indexOperationListeners, namedWriteableRegistry, idFieldDataEnabled, allowExpensiveQueries, expressionResolver, valuesSourceRegistry, recoveryStateFactory);
        success = true;
        return indexService;
    } finally {
        if (success == false) {
            IOUtils.closeWhileHandlingException(queryCache, indexAnalyzers);
        }
    }
}

Example 4 with SimilarityService

use of org.opensearch.index.similarity.SimilarityService in project OpenSearch by opensearch-project.

the class IndexService method createShard.

public synchronized IndexShard createShard(final ShardRouting routing, final Consumer<ShardId> globalCheckpointSyncer, final RetentionLeaseSyncer retentionLeaseSyncer) throws IOException {
    Objects.requireNonNull(retentionLeaseSyncer);
    /*
         * TODO: we execute this in parallel but it's a synced method. Yet, we might
         * be able to serialize the execution via the cluster state in the future. for now we just
         * keep it synced.
         */
    if (closed.get()) {
        throw new IllegalStateException("Can't create shard " + routing.shardId() + ", closed");
    }
    final Settings indexSettings = this.indexSettings.getSettings();
    final ShardId shardId = routing.shardId();
    boolean success = false;
    Store store = null;
    IndexShard indexShard = null;
    ShardLock lock = null;
    try {
        lock = nodeEnv.shardLock(shardId, "starting shard", TimeUnit.SECONDS.toMillis(5));
        eventListener.beforeIndexShardCreated(shardId, indexSettings);
        ShardPath path;
        try {
            path = ShardPath.loadShardPath(logger, nodeEnv, shardId, this.indexSettings.customDataPath());
        } catch (IllegalStateException ex) {
            logger.warn("{} failed to load shard path, trying to remove leftover", shardId);
            try {
                ShardPath.deleteLeftoverShardDirectory(logger, nodeEnv, lock, this.indexSettings);
                path = ShardPath.loadShardPath(logger, nodeEnv, shardId, this.indexSettings.customDataPath());
            } catch (Exception inner) {
                ex.addSuppressed(inner);
                throw ex;
            }
        }
        if (path == null) {
            // TODO: we should, instead, hold a "bytes reserved" of how large we anticipate this shard will be, e.g. for a shard
            // that's being relocated/replicated we know how large it will become once it's done copying:
            // Count up how many shards are currently on each data path:
            Map<Path, Integer> dataPathToShardCount = new HashMap<>();
            for (IndexShard shard : this) {
                Path dataPath = shard.shardPath().getRootStatePath();
                Integer curCount = dataPathToShardCount.get(dataPath);
                if (curCount == null) {
                    curCount = 0;
                }
                dataPathToShardCount.put(dataPath, curCount + 1);
            }
            path = ShardPath.selectNewPathForShard(nodeEnv, shardId, this.indexSettings, routing.getExpectedShardSize() == ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE ? getAvgShardSizeInBytes() : routing.getExpectedShardSize(), dataPathToShardCount);
            logger.debug("{} creating using a new path [{}]", shardId, path);
        } else {
            logger.debug("{} creating using an existing path [{}]", shardId, path);
        }
        if (shards.containsKey(shardId.id())) {
            throw new IllegalStateException(shardId + " already exists");
        }
        logger.debug("creating shard_id {}", shardId);
        // if we are on a shared FS we only own the shard (ie. we can safely delete it) if we are the primary.
        final Engine.Warmer engineWarmer = (reader) -> {
            IndexShard shard = getShardOrNull(shardId.getId());
            if (shard != null) {
                warmer.warm(reader, shard, IndexService.this.indexSettings);
            }
        };
        Directory directory = directoryFactory.newDirectory(this.indexSettings, path);
        store = new Store(shardId, this.indexSettings, directory, lock, new StoreCloseListener(shardId, () -> eventListener.onStoreClosed(shardId)));
        eventListener.onStoreCreated(shardId);
        indexShard = new IndexShard(routing, this.indexSettings, path, store, indexSortSupplier, indexCache, mapperService, similarityService, engineFactory, engineConfigFactory, eventListener, readerWrapper, threadPool, bigArrays, engineWarmer, searchOperationListeners, indexingOperationListeners, () -> globalCheckpointSyncer.accept(shardId), retentionLeaseSyncer, circuitBreakerService);
        eventListener.indexShardStateChanged(indexShard, null, indexShard.state(), "shard created");
        eventListener.afterIndexShardCreated(indexShard);
        shards = newMapBuilder(shards).put(shardId.id(), indexShard).immutableMap();
        success = true;
        return indexShard;
    } catch (ShardLockObtainFailedException e) {
        throw new IOException("failed to obtain in-memory shard lock", e);
    } finally {
        if (success == false) {
            if (lock != null) {
                IOUtils.closeWhileHandlingException(lock);
            }
            closeShard("initialization failed", shardId, indexShard, store, eventListener);
        }
    }
}

Example 5 with SimilarityService

use of org.opensearch.index.similarity.SimilarityService in project OpenSearch by opensearch-project.

the class IndexShardTestCase method newShard.

/**
 * creates a new initializing shard.
 * @param routing                       shard routing to use
 * @param shardPath                     path to use for shard data
 * @param indexMetadata                 indexMetadata for the shard, including any mapping
 * @param storeProvider                 an optional custom store provider to use. If null a default file based store will be created
 * @param indexReaderWrapper            an optional wrapper to be used during search
 * @param globalCheckpointSyncer        callback for syncing global checkpoints
 * @param indexEventListener            index event listener
 * @param listeners                     an optional set of listeners to add to the shard
 */
protected IndexShard newShard(ShardRouting routing, ShardPath shardPath, IndexMetadata indexMetadata, @Nullable CheckedFunction<IndexSettings, Store, IOException> storeProvider, @Nullable CheckedFunction<DirectoryReader, DirectoryReader, IOException> indexReaderWrapper, @Nullable EngineFactory engineFactory, @Nullable EngineConfigFactory engineConfigFactory, Runnable globalCheckpointSyncer, RetentionLeaseSyncer retentionLeaseSyncer, IndexEventListener indexEventListener, IndexingOperationListener... listeners) throws IOException {
    final Settings nodeSettings = Settings.builder().put("node.name", routing.currentNodeId()).build();
    final IndexSettings indexSettings = new IndexSettings(indexMetadata, nodeSettings);
    final IndexShard indexShard;
    if (storeProvider == null) {
        storeProvider = is -> createStore(is, shardPath);
    }
    final Store store = storeProvider.apply(indexSettings);
    boolean success = false;
    try {
        IndexCache indexCache = new IndexCache(indexSettings, new DisabledQueryCache(indexSettings), null);
        MapperService mapperService = MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), indexSettings.getSettings(), "index");
        mapperService.merge(indexMetadata, MapperService.MergeReason.MAPPING_RECOVERY);
        SimilarityService similarityService = new SimilarityService(indexSettings, null, Collections.emptyMap());
        final Engine.Warmer warmer = createTestWarmer(indexSettings);
        ClusterSettings clusterSettings = new ClusterSettings(nodeSettings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
        CircuitBreakerService breakerService = new HierarchyCircuitBreakerService(nodeSettings, Collections.emptyList(), clusterSettings);
        indexShard = new IndexShard(routing, indexSettings, shardPath, store, () -> null, indexCache, mapperService, similarityService, engineFactory, engineConfigFactory, indexEventListener, indexReaderWrapper, threadPool, BigArrays.NON_RECYCLING_INSTANCE, warmer, Collections.emptyList(), Arrays.asList(listeners), globalCheckpointSyncer, retentionLeaseSyncer, breakerService);
        indexShard.addShardFailureCallback(DEFAULT_SHARD_FAILURE_HANDLER);
        success = true;
    } finally {
        if (success == false) {
            IOUtils.close(store);
        }
    }
    return indexShard;
}