Examples with IndexEventListener org.elasticsearch.index.shard.IndexEventListener used on opensource projects

Example 1 with IndexEventListener

use of org.elasticsearch.index.shard.IndexEventListener in project elasticsearch by elastic.

the class IndicesService method createIndexService.

/**
     * This creates a new IndexService without registering it
     */
private synchronized IndexService createIndexService(final String reason, IndexMetaData indexMetaData, IndicesQueryCache indicesQueryCache, IndicesFieldDataCache indicesFieldDataCache, List<IndexEventListener> builtInListeners, Consumer<ShardId> globalCheckpointSyncer, IndexingOperationListener... indexingOperationListeners) throws IOException {
    final Index index = indexMetaData.getIndex();
    final IndexSettings idxSettings = new IndexSettings(indexMetaData, this.settings, indexScopeSetting);
    logger.debug("creating Index [{}], shards [{}]/[{}{}] - reason [{}]", indexMetaData.getIndex(), idxSettings.getNumberOfShards(), idxSettings.getNumberOfReplicas(), idxSettings.isShadowReplicaIndex() ? "s" : "", reason);
    final IndexModule indexModule = new IndexModule(idxSettings, analysisRegistry);
    for (IndexingOperationListener operationListener : indexingOperationListeners) {
        indexModule.addIndexOperationListener(operationListener);
    }
    pluginsService.onIndexModule(indexModule);
    for (IndexEventListener listener : builtInListeners) {
        indexModule.addIndexEventListener(listener);
    }
    return indexModule.newIndexService(nodeEnv, xContentRegistry, this, circuitBreakerService, bigArrays, threadPool, scriptService, clusterService, client, indicesQueryCache, mapperRegistry, globalCheckpointSyncer, indicesFieldDataCache);
}

Example 2 with IndexEventListener

use of org.elasticsearch.index.shard.IndexEventListener in project elasticsearch by elastic.

the class IndexModuleTests method testOtherServiceBound.

public void testOtherServiceBound() throws IOException {
    final AtomicBoolean atomicBoolean = new AtomicBoolean(false);
    final IndexEventListener eventListener = new IndexEventListener() {

        @Override
        public void beforeIndexRemoved(IndexService indexService, IndexRemovalReason reason) {
            atomicBoolean.set(true);
        }
    };
    IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(index, settings);
    IndexModule module = new IndexModule(indexSettings, new AnalysisRegistry(environment, emptyMap(), emptyMap(), emptyMap(), emptyMap(), emptyMap()));
    module.addIndexEventListener(eventListener);
    IndexService indexService = newIndexService(module);
    IndexSettings x = indexService.getIndexSettings();
    assertEquals(x.getSettings().getAsMap(), indexSettings.getSettings().getAsMap());
    assertEquals(x.getIndex(), index);
    indexService.getIndexEventListener().beforeIndexRemoved(null, null);
    assertTrue(atomicBoolean.get());
    indexService.close("simon says", false);
}

Example 3 with IndexEventListener

use of org.elasticsearch.index.shard.IndexEventListener in project elasticsearch by elastic.

the class CorruptedFileIT method testCorruptFileAndRecover.

/**
     * Tests that we can actually recover from a corruption on the primary given that we have replica shards around.
     */
public void testCorruptFileAndRecover() throws ExecutionException, InterruptedException, IOException {
    int numDocs = scaledRandomIntBetween(100, 1000);
    // have enough space for 3 copies
    internalCluster().ensureAtLeastNumDataNodes(3);
    if (cluster().numDataNodes() == 3) {
        logger.info("--> cluster has [3] data nodes, corrupted primary will be overwritten");
    }
    assertThat(cluster().numDataNodes(), greaterThanOrEqualTo(3));
    assertAcked(prepareCreate("test").setSettings(Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, "1").put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "1").put(MergePolicyConfig.INDEX_MERGE_ENABLED, false).put(MockFSIndexStore.INDEX_CHECK_INDEX_ON_CLOSE_SETTING.getKey(), // no checkindex - we corrupt shards on purpose
    false).put(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING.getKey(), // no translog based flush - it might change the .liv / segments.N files
    new ByteSizeValue(1, ByteSizeUnit.PB))));
    ensureGreen();
    disableAllocation("test");
    IndexRequestBuilder[] builders = new IndexRequestBuilder[numDocs];
    for (int i = 0; i < builders.length; i++) {
        builders[i] = client().prepareIndex("test", "type").setSource("field", "value");
    }
    indexRandom(true, builders);
    ensureGreen();
    assertAllSuccessful(client().admin().indices().prepareFlush().setForce(true).execute().actionGet());
    // we have to flush at least once here since we don't corrupt the translog
    SearchResponse countResponse = client().prepareSearch().setSize(0).get();
    assertHitCount(countResponse, numDocs);
    final int numShards = numShards("test");
    ShardRouting corruptedShardRouting = corruptRandomPrimaryFile();
    logger.info("--> {} corrupted", corruptedShardRouting);
    enableAllocation("test");
    /*
         * we corrupted the primary shard - now lets make sure we never recover from it successfully
         */
    Settings build = Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "2").build();
    client().admin().indices().prepareUpdateSettings("test").setSettings(build).get();
    ClusterHealthResponse health = client().admin().cluster().health(Requests.clusterHealthRequest("test").waitForGreenStatus().timeout(// sometimes due to cluster rebalacing and random settings default timeout is just not enough.
    "5m").waitForNoRelocatingShards(true)).actionGet();
    if (health.isTimedOut()) {
        logger.info("cluster state:\n{}\n{}", client().admin().cluster().prepareState().get().getState(), client().admin().cluster().preparePendingClusterTasks().get());
        assertThat("timed out waiting for green state", health.isTimedOut(), equalTo(false));
    }
    assertThat(health.getStatus(), equalTo(ClusterHealthStatus.GREEN));
    final int numIterations = scaledRandomIntBetween(5, 20);
    for (int i = 0; i < numIterations; i++) {
        SearchResponse response = client().prepareSearch().setSize(numDocs).get();
        assertHitCount(response, numDocs);
    }
    /*
         * now hook into the IndicesService and register a close listener to
         * run the checkindex. if the corruption is still there we will catch it.
         */
    // primary + 2 replicas
    final CountDownLatch latch = new CountDownLatch(numShards * 3);
    final CopyOnWriteArrayList<Exception> exception = new CopyOnWriteArrayList<>();
    final IndexEventListener listener = new IndexEventListener() {

        @Override
        public void afterIndexShardClosed(ShardId sid, @Nullable IndexShard indexShard, Settings indexSettings) {
            if (indexShard != null) {
                Store store = indexShard.store();
                store.incRef();
                try {
                    if (!Lucene.indexExists(store.directory()) && indexShard.state() == IndexShardState.STARTED) {
                        return;
                    }
                    try (CheckIndex checkIndex = new CheckIndex(store.directory())) {
                        BytesStreamOutput os = new BytesStreamOutput();
                        PrintStream out = new PrintStream(os, false, StandardCharsets.UTF_8.name());
                        checkIndex.setInfoStream(out);
                        out.flush();
                        CheckIndex.Status status = checkIndex.checkIndex();
                        if (!status.clean) {
                            logger.warn("check index [failure]\n{}", os.bytes().utf8ToString());
                            throw new IOException("index check failure");
                        }
                    }
                } catch (Exception e) {
                    exception.add(e);
                } finally {
                    store.decRef();
                    latch.countDown();
                }
            }
        }
    };
    for (MockIndexEventListener.TestEventListener eventListener : internalCluster().getDataNodeInstances(MockIndexEventListener.TestEventListener.class)) {
        eventListener.setNewDelegate(listener);
    }
    try {
        client().admin().indices().prepareDelete("test").get();
        latch.await();
        assertThat(exception, empty());
    } finally {
        for (MockIndexEventListener.TestEventListener eventListener : internalCluster().getDataNodeInstances(MockIndexEventListener.TestEventListener.class)) {
            eventListener.setNewDelegate(null);
        }
    }
}

Example 4 with IndexEventListener

use of org.elasticsearch.index.shard.IndexEventListener in project crate by crate.

the class IndicesService method removeIndex.

@Override
public void removeIndex(final Index index, final IndexRemovalReason reason, final String extraInfo) {
    final String indexName = index.getName();
    try {
        final IndexService indexService;
        final IndexEventListener listener;
        synchronized (this) {
            if (hasIndex(index) == false) {
                return;
            }
            LOGGER.debug("[{}] closing ... (reason [{}])", indexName, reason);
            Map<String, IndexService> newIndices = new HashMap<>(indices);
            indexService = newIndices.remove(index.getUUID());
            assert indexService != null : "IndexService is null for index: " + index;
            indices = unmodifiableMap(newIndices);
            listener = indexService.getIndexEventListener();
        }
        listener.beforeIndexRemoved(indexService, reason);
        LOGGER.debug("{} closing index service (reason [{}][{}])", index, reason, extraInfo);
        indexService.close(extraInfo, reason == IndexRemovalReason.DELETED);
        LOGGER.debug("{} closed... (reason [{}][{}])", index, reason, extraInfo);
        final IndexSettings indexSettings = indexService.getIndexSettings();
        listener.afterIndexRemoved(indexService.index(), indexSettings, reason);
        if (reason == IndexRemovalReason.DELETED) {
            // now we are done - try to wipe data on disk if possible
            deleteIndexStore(extraInfo, indexService.index(), indexSettings);
        }
    } catch (Exception e) {
        LOGGER.warn(() -> new ParameterizedMessage("failed to remove index {} ([{}][{}])", index, reason, extraInfo), e);
    }
}

Example 5 with IndexEventListener

use of org.elasticsearch.index.shard.IndexEventListener in project crate by crate.

the class IndexModule method newIndexService.

public IndexService newIndexService(IndexService.IndexCreationContext indexCreationContext, NodeEnvironment environment, NamedXContentRegistry xContentRegistry, IndexService.ShardStoreDeleter shardStoreDeleter, CircuitBreakerService circuitBreakerService, BigArrays bigArrays, ThreadPool threadPool, IndicesQueryCache indicesQueryCache, MapperRegistry mapperRegistry) throws IOException {
    final IndexEventListener eventListener = freeze();
    eventListener.beforeIndexCreated(indexSettings.getIndex(), indexSettings.getSettings());
    final IndexStorePlugin.DirectoryFactory directoryFactory = getDirectoryFactory(indexSettings, directoryFactories);
    final QueryCache queryCache;
    if (indexSettings.getValue(INDEX_QUERY_CACHE_ENABLED_SETTING)) {
        BiFunction<IndexSettings, IndicesQueryCache, QueryCache> queryCacheProvider = forceQueryCacheProvider.get();
        if (queryCacheProvider == null) {
            queryCache = new IndexQueryCache(indexSettings, indicesQueryCache);
        } else {
            queryCache = queryCacheProvider.apply(indexSettings, indicesQueryCache);
        }
    } else {
        queryCache = new DisabledQueryCache(indexSettings);
    }
    return new IndexService(indexSettings, indexCreationContext, environment, xContentRegistry, shardStoreDeleter, analysisRegistry, engineFactory, circuitBreakerService, bigArrays, threadPool, queryCache, directoryFactory, eventListener, mapperRegistry, indexOperationListeners);
}