Examples with Index org.opensearch.index.Index used on opensource projects

Example 21 with Index

use of org.opensearch.index.Index in project OpenSearch by opensearch-project.

the class IndicesService method doStop.

@Override
protected void doStop() {
    ThreadPool.terminate(danglingIndicesThreadPoolExecutor, 10, TimeUnit.SECONDS);
    ExecutorService indicesStopExecutor = Executors.newFixedThreadPool(5, daemonThreadFactory(settings, "indices_shutdown"));
    // Copy indices because we modify it asynchronously in the body of the loop
    final Set<Index> indices = this.indices.values().stream().map(s -> s.index()).collect(Collectors.toSet());
    final CountDownLatch latch = new CountDownLatch(indices.size());
    for (final Index index : indices) {
        indicesStopExecutor.execute(() -> {
            try {
                removeIndex(index, IndexRemovalReason.NO_LONGER_ASSIGNED, "shutdown");
            } finally {
                latch.countDown();
            }
        });
    }
    try {
        if (latch.await(shardsClosedTimeout.seconds(), TimeUnit.SECONDS) == false) {
            logger.warn("Not all shards are closed yet, waited {}sec - stopping service", shardsClosedTimeout.seconds());
        }
    } catch (InterruptedException e) {
    // ignore
    } finally {
        indicesStopExecutor.shutdown();
    }
}

Example 22 with Index

use of org.opensearch.index.Index in project OpenSearch by opensearch-project.

the class IndicesClusterStateService method deleteIndices.

/**
 * Deletes indices (with shard data).
 *
 * @param event cluster change event
 */
private void deleteIndices(final ClusterChangedEvent event) {
    final ClusterState previousState = event.previousState();
    final ClusterState state = event.state();
    final String localNodeId = state.nodes().getLocalNodeId();
    assert localNodeId != null;
    for (Index index : event.indicesDeleted()) {
        if (logger.isDebugEnabled()) {
            logger.debug("[{}] cleaning index, no longer part of the metadata", index);
        }
        AllocatedIndex<? extends Shard> indexService = indicesService.indexService(index);
        final IndexSettings indexSettings;
        if (indexService != null) {
            indexSettings = indexService.getIndexSettings();
            indicesService.removeIndex(index, DELETED, "index no longer part of the metadata");
        } else if (previousState.metadata().hasIndex(index)) {
            // The deleted index was part of the previous cluster state, but not loaded on the local node
            final IndexMetadata metadata = previousState.metadata().index(index);
            indexSettings = new IndexSettings(metadata, settings);
            indicesService.deleteUnassignedIndex("deleted index was not assigned to local node", metadata, state);
        } else {
            // previous cluster state is not initialized/recovered.
            assert state.metadata().indexGraveyard().containsIndex(index) || previousState.blocks().hasGlobalBlock(GatewayService.STATE_NOT_RECOVERED_BLOCK);
            final IndexMetadata metadata = indicesService.verifyIndexIsDeleted(index, event.state());
            if (metadata != null) {
                indexSettings = new IndexSettings(metadata, settings);
            } else {
                indexSettings = null;
            }
        }
        if (indexSettings != null) {
            threadPool.generic().execute(new AbstractRunnable() {

                @Override
                public void onFailure(Exception e) {
                    logger.warn(() -> new ParameterizedMessage("[{}] failed to complete pending deletion for index", index), e);
                }

                @Override
                protected void doRun() throws Exception {
                    try {
                        // we are waiting until we can lock the index / all shards on the node and then we ack the delete of the store
                        // to the master. If we can't acquire the locks here immediately there might be a shard of this index still
                        // holding on to the lock due to a "currently canceled recovery" or so. The shard will delete itself BEFORE the
                        // lock is released so it's guaranteed to be deleted by the time we get the lock
                        indicesService.processPendingDeletes(index, indexSettings, new TimeValue(30, TimeUnit.MINUTES));
                    } catch (ShardLockObtainFailedException exc) {
                        logger.warn("[{}] failed to lock all shards for index - timed out after 30 seconds", index);
                    } catch (InterruptedException e) {
                        logger.warn("[{}] failed to lock all shards for index - interrupted", index);
                    }
                }
            });
        }
    }
}

Example 23 with Index

use of org.opensearch.index.Index in project OpenSearch by opensearch-project.

the class IndicesClusterStateService method createIndices.

private void createIndices(final ClusterState state) {
    // we only create indices for shards that are allocated
    RoutingNode localRoutingNode = state.getRoutingNodes().node(state.nodes().getLocalNodeId());
    if (localRoutingNode == null) {
        return;
    }
    // create map of indices to create with shards to fail if index creation fails
    final Map<Index, List<ShardRouting>> indicesToCreate = new HashMap<>();
    for (ShardRouting shardRouting : localRoutingNode) {
        if (failedShardsCache.containsKey(shardRouting.shardId()) == false) {
            final Index index = shardRouting.index();
            if (indicesService.indexService(index) == null) {
                indicesToCreate.computeIfAbsent(index, k -> new ArrayList<>()).add(shardRouting);
            }
        }
    }
    for (Map.Entry<Index, List<ShardRouting>> entry : indicesToCreate.entrySet()) {
        final Index index = entry.getKey();
        final IndexMetadata indexMetadata = state.metadata().index(index);
        logger.debug("[{}] creating index", index);
        AllocatedIndex<? extends Shard> indexService = null;
        try {
            indexService = indicesService.createIndex(indexMetadata, buildInIndexListener, true);
            if (indexService.updateMapping(null, indexMetadata) && sendRefreshMapping) {
                nodeMappingRefreshAction.nodeMappingRefresh(state.nodes().getMasterNode(), new NodeMappingRefreshAction.NodeMappingRefreshRequest(indexMetadata.getIndex().getName(), indexMetadata.getIndexUUID(), state.nodes().getLocalNodeId()));
            }
        } catch (Exception e) {
            final String failShardReason;
            if (indexService == null) {
                failShardReason = "failed to create index";
            } else {
                failShardReason = "failed to update mapping for index";
                indicesService.removeIndex(index, FAILURE, "removing index (mapping update failed)");
            }
            for (ShardRouting shardRouting : entry.getValue()) {
                sendFailShard(shardRouting, failShardReason, e, state);
            }
        }
    }
}

Example 24 with Index

use of org.opensearch.index.Index in project OpenSearch by opensearch-project.

the class NodeIndicesStats method writeTo.

@Override
public void writeTo(StreamOutput out) throws IOException {
    stats.writeTo(out);
    out.writeBoolean(statsByShard != null);
    if (statsByShard != null) {
        out.writeVInt(statsByShard.size());
        for (Map.Entry<Index, List<IndexShardStats>> entry : statsByShard.entrySet()) {
            entry.getKey().writeTo(out);
            out.writeVInt(entry.getValue().size());
            for (IndexShardStats indexShardStats : entry.getValue()) {
                indexShardStats.writeTo(out);
            }
        }
    }
}

Example 25 with Index

use of org.opensearch.index.Index in project OpenSearch by opensearch-project.

the class ShardLimitValidator method validateShardLimit.

/**
 * Validates whether a list of indices can be opened without going over the cluster shard limit.  Only counts indices which are
 * currently closed and will be opened, ignores indices which are already open.
 *
 * @param currentState The current cluster state.
 * @param indicesToOpen The indices which are to be opened.
 * @throws ValidationException If this operation would take the cluster over the limit and enforcement is enabled.
 */
public void validateShardLimit(ClusterState currentState, Index[] indicesToOpen) {
    int shardsToOpen = Arrays.stream(indicesToOpen).filter(index -> !systemIndices.validateSystemIndex(index.getName())).filter(index -> currentState.metadata().index(index).getState().equals(IndexMetadata.State.CLOSE)).mapToInt(index -> getTotalShardCount(currentState, index)).sum();
    Optional<String> error = checkShardLimit(shardsToOpen, currentState);
    if (error.isPresent()) {
        ValidationException ex = new ValidationException();
        ex.addValidationError(error.get());
        throw ex;
    }
}