Examples with IndicesFieldDataCache org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache used on opensource projects

Example 6 with IndicesFieldDataCache

use of org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache in project elasticsearch by elastic.

the class RandomExceptionCircuitBreakerIT method testBreakerWithRandomExceptions.

public void testBreakerWithRandomExceptions() throws IOException, InterruptedException, ExecutionException {
    for (NodeStats node : client().admin().cluster().prepareNodesStats().clear().setBreaker(true).execute().actionGet().getNodes()) {
        assertThat("Breaker is not set to 0", node.getBreaker().getStats(CircuitBreaker.FIELDDATA).getEstimated(), equalTo(0L));
    }
    String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties").startObject("test-str").field("type", "keyword").field("doc_values", randomBoolean()).endObject().startObject("test-num").field("type", randomFrom(Arrays.asList("float", "long", "double", "short", "integer"))).endObject().endObject().endObject().endObject().string();
    final double topLevelRate;
    final double lowLevelRate;
    if (frequently()) {
        if (randomBoolean()) {
            if (randomBoolean()) {
                lowLevelRate = 1.0 / between(2, 10);
                topLevelRate = 0.0d;
            } else {
                topLevelRate = 1.0 / between(2, 10);
                lowLevelRate = 0.0d;
            }
        } else {
            lowLevelRate = 1.0 / between(2, 10);
            topLevelRate = 1.0 / between(2, 10);
        }
    } else {
        // rarely no exception
        topLevelRate = 0d;
        lowLevelRate = 0d;
    }
    Settings.Builder settings = Settings.builder().put(indexSettings()).put(EXCEPTION_TOP_LEVEL_RATIO_KEY, topLevelRate).put(EXCEPTION_LOW_LEVEL_RATIO_KEY, lowLevelRate).put(MockEngineSupport.WRAP_READER_RATIO.getKey(), 1.0d);
    logger.info("creating index: [test] using settings: [{}]", settings.build().getAsMap());
    CreateIndexResponse response = client().admin().indices().prepareCreate("test").setSettings(settings).addMapping("type", mapping, XContentType.JSON).execute().actionGet();
    final int numDocs;
    if (response.isShardsAcked() == false) {
        /* some seeds just won't let you create the index at all and we enter a ping-pong mode
             * trying one node after another etc. that is ok but we need to make sure we don't wait
             * forever when indexing documents so we set numDocs = 1 and expect all shards to fail
             * when we search below.*/
        if (response.isAcknowledged()) {
            logger.info("Index creation timed out waiting for primaries to start - only index one doc and expect searches to fail");
        } else {
            logger.info("Index creation failed - only index one doc and expect searches to fail");
        }
        numDocs = 1;
    } else {
        numDocs = between(10, 100);
    }
    for (int i = 0; i < numDocs; i++) {
        try {
            client().prepareIndex("test", "type", "" + i).setTimeout(TimeValue.timeValueSeconds(1)).setSource("test-str", randomUnicodeOfLengthBetween(5, 25), "test-num", i).get();
        } catch (ElasticsearchException ex) {
        }
    }
    logger.info("Start Refresh");
    // don't assert on failures here
    RefreshResponse refreshResponse = client().admin().indices().prepareRefresh("test").execute().get();
    final boolean refreshFailed = refreshResponse.getShardFailures().length != 0 || refreshResponse.getFailedShards() != 0;
    logger.info("Refresh failed: [{}] numShardsFailed: [{}], shardFailuresLength: [{}], successfulShards: [{}], totalShards: [{}] ", refreshFailed, refreshResponse.getFailedShards(), refreshResponse.getShardFailures().length, refreshResponse.getSuccessfulShards(), refreshResponse.getTotalShards());
    final int numSearches = scaledRandomIntBetween(50, 150);
    NodesStatsResponse resp = client().admin().cluster().prepareNodesStats().clear().setBreaker(true).execute().actionGet();
    for (NodeStats stats : resp.getNodes()) {
        assertThat("Breaker is set to 0", stats.getBreaker().getStats(CircuitBreaker.FIELDDATA).getEstimated(), equalTo(0L));
    }
    for (int i = 0; i < numSearches; i++) {
        SearchRequestBuilder searchRequestBuilder = client().prepareSearch().setQuery(QueryBuilders.matchAllQuery());
        if (random().nextBoolean()) {
            searchRequestBuilder.addSort("test-str", SortOrder.ASC);
        }
        searchRequestBuilder.addSort("test-num", SortOrder.ASC);
        boolean success = false;
        try {
            // Sort by the string and numeric fields, to load them into field data
            searchRequestBuilder.get();
            success = true;
        } catch (SearchPhaseExecutionException ex) {
            logger.info("expected SearchPhaseException: [{}]", ex.getMessage());
        }
        if (frequently()) {
            // Now, clear the cache and check that the circuit breaker has been
            // successfully set back to zero. If there is a bug in the circuit
            // breaker adjustment code, it should show up here by the breaker
            // estimate being either positive or negative.
            // make sure all shards are there - there could be shards that are still starting up.
            ensureGreen("test");
            assertAllSuccessful(client().admin().indices().prepareClearCache("test").setFieldDataCache(true).execute().actionGet());
            // Since .cleanUp() is no longer called on cache clear, we need to call it on each node manually
            for (String node : internalCluster().getNodeNames()) {
                final IndicesFieldDataCache fdCache = internalCluster().getInstance(IndicesService.class, node).getIndicesFieldDataCache();
                // Clean up the cache, ensuring that entries' listeners have been called
                fdCache.getCache().refresh();
            }
            NodesStatsResponse nodeStats = client().admin().cluster().prepareNodesStats().clear().setBreaker(true).execute().actionGet();
            for (NodeStats stats : nodeStats.getNodes()) {
                assertThat("Breaker reset to 0 last search success: " + success + " mapping: " + mapping, stats.getBreaker().getStats(CircuitBreaker.FIELDDATA).getEstimated(), equalTo(0L));
            }
        }
    }
}

Example 7 with IndicesFieldDataCache

use of org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache in project elasticsearch by elastic.

the class IndexShardTestCase method newShard.

/**
     * creates a new initializing shard.
     *
     * @param routing              shard routing to use
     * @param shardPath            path to use for shard data
     * @param indexMetaData        indexMetaData for the shard, including any mapping
     * @param indexSearcherWrapper an optional wrapper to be used during searchers
     * @param listeners            an optional set of listeners to add to the shard
     */
protected IndexShard newShard(ShardRouting routing, ShardPath shardPath, IndexMetaData indexMetaData, @Nullable IndexSearcherWrapper indexSearcherWrapper, Runnable globalCheckpointSyncer, @Nullable EngineFactory engineFactory, IndexingOperationListener... listeners) throws IOException {
    final Settings nodeSettings = Settings.builder().put("node.name", routing.currentNodeId()).build();
    final IndexSettings indexSettings = new IndexSettings(indexMetaData, nodeSettings);
    final IndexShard indexShard;
    final Store store = createStore(indexSettings, shardPath);
    boolean success = false;
    try {
        IndexCache indexCache = new IndexCache(indexSettings, new DisabledQueryCache(indexSettings), null);
        MapperService mapperService = MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), indexSettings.getSettings());
        mapperService.merge(indexMetaData, MapperService.MergeReason.MAPPING_RECOVERY, true);
        SimilarityService similarityService = new SimilarityService(indexSettings, Collections.emptyMap());
        final IndexEventListener indexEventListener = new IndexEventListener() {
        };
        final Engine.Warmer warmer = searcher -> {
        };
        IndicesFieldDataCache indicesFieldDataCache = new IndicesFieldDataCache(nodeSettings, new IndexFieldDataCache.Listener() {
        });
        IndexFieldDataService indexFieldDataService = new IndexFieldDataService(indexSettings, indicesFieldDataCache, new NoneCircuitBreakerService(), mapperService);
        indexShard = new IndexShard(routing, indexSettings, shardPath, store, indexCache, mapperService, similarityService, indexFieldDataService, engineFactory, indexEventListener, indexSearcherWrapper, threadPool, BigArrays.NON_RECYCLING_INSTANCE, warmer, globalCheckpointSyncer, Collections.emptyList(), Arrays.asList(listeners));
        success = true;
    } finally {
        if (success == false) {
            IOUtils.close(store);
        }
    }
    return indexShard;
}