Examples with IndexCache org.elasticsearch.index.cache.IndexCache used on opensource projects

Example 1 with IndexCache

use of org.elasticsearch.index.cache.IndexCache in project crate by crate.

the class LuceneQueryBuilderTest method prepare.

@Before
public void prepare() throws Exception {
    DocTableInfo users = TestingTableInfo.builder(new TableIdent(null, "users"), null).add("name", DataTypes.STRING).add("x", DataTypes.INTEGER).add("d", DataTypes.DOUBLE).add("d_array", new ArrayType(DataTypes.DOUBLE)).add("y_array", new ArrayType(DataTypes.LONG)).add("shape", DataTypes.GEO_SHAPE).add("point", DataTypes.GEO_POINT).build();
    TableRelation usersTr = new TableRelation(users);
    sources = ImmutableMap.of(new QualifiedName("users"), usersTr);
    expressions = new SqlExpressions(sources, usersTr);
    builder = new LuceneQueryBuilder(expressions.getInstance(Functions.class));
    indexCache = mock(IndexCache.class, Answers.RETURNS_MOCKS.get());
    Path tempDir = createTempDir();
    Settings indexSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).put("path.home", tempDir).build();
    Index index = new Index(users.ident().indexName());
    when(indexCache.indexSettings()).thenReturn(indexSettings);
    AnalysisService analysisService = createAnalysisService(indexSettings, index);
    mapperService = createMapperService(index, indexSettings, analysisService);
    // @formatter:off
    XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().startObject("default").startObject("properties").startObject("name").field("type", "string").endObject().startObject("x").field("type", "integer").endObject().startObject("d").field("type", "double").endObject().startObject("point").field("type", "geo_point").endObject().startObject("shape").field("type", "geo_shape").endObject().startObject("d_array").field("type", "array").startObject("inner").field("type", "double").endObject().endObject().startObject("y_array").field("type", "array").startObject("inner").field("type", "integer").endObject().endObject().endObject().endObject().endObject();
    // @formatter:on
    mapperService.merge("default", new CompressedXContent(xContentBuilder.bytes()), MapperService.MergeReason.MAPPING_UPDATE, true);
    indexFieldDataService = mock(IndexFieldDataService.class);
    IndexFieldData geoFieldData = mock(IndexGeoPointFieldData.class);
    when(geoFieldData.getFieldNames()).thenReturn(new MappedFieldType.Names("point"));
    when(indexFieldDataService.getForField(mapperService.smartNameFieldType("point"))).thenReturn(geoFieldData);
}

Example 2 with IndexCache

use of org.elasticsearch.index.cache.IndexCache in project elasticsearch by elastic.

the class IndexService method createShard.

public synchronized IndexShard createShard(ShardRouting routing) throws IOException {
    final boolean primary = routing.primary();
    /*
         * TODO: we execute this in parallel but it's a synced method. Yet, we might
         * be able to serialize the execution via the cluster state in the future. for now we just
         * keep it synced.
         */
    if (closed.get()) {
        throw new IllegalStateException("Can't create shard " + routing.shardId() + ", closed");
    }
    final Settings indexSettings = this.indexSettings.getSettings();
    final ShardId shardId = routing.shardId();
    boolean success = false;
    Store store = null;
    IndexShard indexShard = null;
    ShardLock lock = null;
    try {
        lock = nodeEnv.shardLock(shardId, TimeUnit.SECONDS.toMillis(5));
        eventListener.beforeIndexShardCreated(shardId, indexSettings);
        ShardPath path;
        try {
            path = ShardPath.loadShardPath(logger, nodeEnv, shardId, this.indexSettings);
        } catch (IllegalStateException ex) {
            logger.warn("{} failed to load shard path, trying to remove leftover", shardId);
            try {
                ShardPath.deleteLeftoverShardDirectory(logger, nodeEnv, lock, this.indexSettings);
                path = ShardPath.loadShardPath(logger, nodeEnv, shardId, this.indexSettings);
            } catch (Exception inner) {
                ex.addSuppressed(inner);
                throw ex;
            }
        }
        if (path == null) {
            // TODO: we should, instead, hold a "bytes reserved" of how large we anticipate this shard will be, e.g. for a shard
            // that's being relocated/replicated we know how large it will become once it's done copying:
            // Count up how many shards are currently on each data path:
            Map<Path, Integer> dataPathToShardCount = new HashMap<>();
            for (IndexShard shard : this) {
                Path dataPath = shard.shardPath().getRootStatePath();
                Integer curCount = dataPathToShardCount.get(dataPath);
                if (curCount == null) {
                    curCount = 0;
                }
                dataPathToShardCount.put(dataPath, curCount + 1);
            }
            path = ShardPath.selectNewPathForShard(nodeEnv, shardId, this.indexSettings, routing.getExpectedShardSize() == ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE ? getAvgShardSizeInBytes() : routing.getExpectedShardSize(), dataPathToShardCount);
            logger.debug("{} creating using a new path [{}]", shardId, path);
        } else {
            logger.debug("{} creating using an existing path [{}]", shardId, path);
        }
        if (shards.containsKey(shardId.id())) {
            throw new IllegalStateException(shardId + " already exists");
        }
        logger.debug("creating shard_id {}", shardId);
        // if we are on a shared FS we only own the shard (ie. we can safely delete it) if we are the primary.
        final boolean canDeleteShardContent = this.indexSettings.isOnSharedFilesystem() == false || (primary && this.indexSettings.isOnSharedFilesystem());
        final Engine.Warmer engineWarmer = (searcher) -> {
            IndexShard shard = getShardOrNull(shardId.getId());
            if (shard != null) {
                warmer.warm(searcher, shard, IndexService.this.indexSettings);
            }
        };
        store = new Store(shardId, this.indexSettings, indexStore.newDirectoryService(path), lock, new StoreCloseListener(shardId, canDeleteShardContent, () -> eventListener.onStoreClosed(shardId)));
        if (useShadowEngine(primary, this.indexSettings)) {
            indexShard = new ShadowIndexShard(routing, this.indexSettings, path, store, indexCache, mapperService, similarityService, indexFieldData, engineFactory, eventListener, searcherWrapper, threadPool, bigArrays, engineWarmer, searchOperationListeners);
        // no indexing listeners - shadow  engines don't index
        } else {
            indexShard = new IndexShard(routing, this.indexSettings, path, store, indexCache, mapperService, similarityService, indexFieldData, engineFactory, eventListener, searcherWrapper, threadPool, bigArrays, engineWarmer, () -> globalCheckpointSyncer.accept(shardId), searchOperationListeners, indexingOperationListeners);
        }
        eventListener.indexShardStateChanged(indexShard, null, indexShard.state(), "shard created");
        eventListener.afterIndexShardCreated(indexShard);
        shards = newMapBuilder(shards).put(shardId.id(), indexShard).immutableMap();
        success = true;
        return indexShard;
    } catch (ShardLockObtainFailedException e) {
        throw new IOException("failed to obtain in-memory shard lock", e);
    } finally {
        if (success == false) {
            if (lock != null) {
                IOUtils.closeWhileHandlingException(lock);
            }
            closeShard("initialization failed", shardId, indexShard, store, eventListener);
        }
    }
}

Example 3 with IndexCache

use of org.elasticsearch.index.cache.IndexCache in project elasticsearch by elastic.

the class IndexShardTestCase method newShard.

/**
     * creates a new initializing shard.
     *
     * @param routing              shard routing to use
     * @param shardPath            path to use for shard data
     * @param indexMetaData        indexMetaData for the shard, including any mapping
     * @param indexSearcherWrapper an optional wrapper to be used during searchers
     * @param listeners            an optional set of listeners to add to the shard
     */
protected IndexShard newShard(ShardRouting routing, ShardPath shardPath, IndexMetaData indexMetaData, @Nullable IndexSearcherWrapper indexSearcherWrapper, Runnable globalCheckpointSyncer, @Nullable EngineFactory engineFactory, IndexingOperationListener... listeners) throws IOException {
    final Settings nodeSettings = Settings.builder().put("node.name", routing.currentNodeId()).build();
    final IndexSettings indexSettings = new IndexSettings(indexMetaData, nodeSettings);
    final IndexShard indexShard;
    final Store store = createStore(indexSettings, shardPath);
    boolean success = false;
    try {
        IndexCache indexCache = new IndexCache(indexSettings, new DisabledQueryCache(indexSettings), null);
        MapperService mapperService = MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), indexSettings.getSettings());
        mapperService.merge(indexMetaData, MapperService.MergeReason.MAPPING_RECOVERY, true);
        SimilarityService similarityService = new SimilarityService(indexSettings, Collections.emptyMap());
        final IndexEventListener indexEventListener = new IndexEventListener() {
        };
        final Engine.Warmer warmer = searcher -> {
        };
        IndicesFieldDataCache indicesFieldDataCache = new IndicesFieldDataCache(nodeSettings, new IndexFieldDataCache.Listener() {
        });
        IndexFieldDataService indexFieldDataService = new IndexFieldDataService(indexSettings, indicesFieldDataCache, new NoneCircuitBreakerService(), mapperService);
        indexShard = new IndexShard(routing, indexSettings, shardPath, store, indexCache, mapperService, similarityService, indexFieldDataService, engineFactory, indexEventListener, indexSearcherWrapper, threadPool, BigArrays.NON_RECYCLING_INSTANCE, warmer, globalCheckpointSyncer, Collections.emptyList(), Arrays.asList(listeners));
        success = true;
    } finally {
        if (success == false) {
            IOUtils.close(store);
        }
    }
    return indexShard;
}