Examples with Index org.elasticsearch.index.Index used on opensource projects

Example 41 with Index

use of org.elasticsearch.index.Index in project elasticsearch by elastic.

the class MetaDataUpdateSettingsService method updateSettings.

public void updateSettings(final UpdateSettingsClusterStateUpdateRequest request, final ActionListener<ClusterStateUpdateResponse> listener) {
    final Settings normalizedSettings = Settings.builder().put(request.settings()).normalizePrefix(IndexMetaData.INDEX_SETTING_PREFIX).build();
    Settings.Builder settingsForClosedIndices = Settings.builder();
    Settings.Builder settingsForOpenIndices = Settings.builder();
    Settings.Builder skipppedSettings = Settings.builder();
    indexScopedSettings.validate(normalizedSettings);
    // never allow to change the number of shards
    for (Map.Entry<String, String> entry : normalizedSettings.getAsMap().entrySet()) {
        if (entry.getKey().equals(IndexMetaData.SETTING_NUMBER_OF_SHARDS)) {
            listener.onFailure(new IllegalArgumentException("can't change the number of shards for an index"));
            return;
        }
        Setting setting = indexScopedSettings.get(entry.getKey());
        // we already validated the normalized settings
        assert setting != null;
        settingsForClosedIndices.put(entry.getKey(), entry.getValue());
        if (setting.isDynamic()) {
            settingsForOpenIndices.put(entry.getKey(), entry.getValue());
        } else {
            skipppedSettings.put(entry.getKey(), entry.getValue());
        }
    }
    final Settings skippedSettigns = skipppedSettings.build();
    final Settings closedSettings = settingsForClosedIndices.build();
    final Settings openSettings = settingsForOpenIndices.build();
    final boolean preserveExisting = request.isPreserveExisting();
    clusterService.submitStateUpdateTask("update-settings", new AckedClusterStateUpdateTask<ClusterStateUpdateResponse>(Priority.URGENT, request, wrapPreservingContext(listener)) {

        @Override
        protected ClusterStateUpdateResponse newResponse(boolean acknowledged) {
            return new ClusterStateUpdateResponse(acknowledged);
        }

        @Override
        public ClusterState execute(ClusterState currentState) {
            RoutingTable.Builder routingTableBuilder = RoutingTable.builder(currentState.routingTable());
            MetaData.Builder metaDataBuilder = MetaData.builder(currentState.metaData());
            // allow to change any settings to a close index, and only allow dynamic settings to be changed
            // on an open index
            Set<Index> openIndices = new HashSet<>();
            Set<Index> closeIndices = new HashSet<>();
            final String[] actualIndices = new String[request.indices().length];
            for (int i = 0; i < request.indices().length; i++) {
                Index index = request.indices()[i];
                actualIndices[i] = index.getName();
                final IndexMetaData metaData = currentState.metaData().getIndexSafe(index);
                if (metaData.getState() == IndexMetaData.State.OPEN) {
                    openIndices.add(index);
                } else {
                    closeIndices.add(index);
                }
            }
            if (!skippedSettigns.isEmpty() && !openIndices.isEmpty()) {
                throw new IllegalArgumentException(String.format(Locale.ROOT, "Can't update non dynamic settings [%s] for open indices %s", skippedSettigns.getAsMap().keySet(), openIndices));
            }
            int updatedNumberOfReplicas = openSettings.getAsInt(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, -1);
            if (updatedNumberOfReplicas != -1 && preserveExisting == false) {
                // we do *not* update the in sync allocation ids as they will be removed upon the first index
                // operation which make these copies stale
                // TODO: update the list once the data is deleted by the node?
                routingTableBuilder.updateNumberOfReplicas(updatedNumberOfReplicas, actualIndices);
                metaDataBuilder.updateNumberOfReplicas(updatedNumberOfReplicas, actualIndices);
                logger.info("updating number_of_replicas to [{}] for indices {}", updatedNumberOfReplicas, actualIndices);
            }
            ClusterBlocks.Builder blocks = ClusterBlocks.builder().blocks(currentState.blocks());
            maybeUpdateClusterBlock(actualIndices, blocks, IndexMetaData.INDEX_READ_ONLY_BLOCK, IndexMetaData.INDEX_READ_ONLY_SETTING, openSettings);
            maybeUpdateClusterBlock(actualIndices, blocks, IndexMetaData.INDEX_METADATA_BLOCK, IndexMetaData.INDEX_BLOCKS_METADATA_SETTING, openSettings);
            maybeUpdateClusterBlock(actualIndices, blocks, IndexMetaData.INDEX_WRITE_BLOCK, IndexMetaData.INDEX_BLOCKS_WRITE_SETTING, openSettings);
            maybeUpdateClusterBlock(actualIndices, blocks, IndexMetaData.INDEX_READ_BLOCK, IndexMetaData.INDEX_BLOCKS_READ_SETTING, openSettings);
            if (!openIndices.isEmpty()) {
                for (Index index : openIndices) {
                    IndexMetaData indexMetaData = metaDataBuilder.getSafe(index);
                    Settings.Builder updates = Settings.builder();
                    Settings.Builder indexSettings = Settings.builder().put(indexMetaData.getSettings());
                    if (indexScopedSettings.updateDynamicSettings(openSettings, indexSettings, updates, index.getName())) {
                        if (preserveExisting) {
                            indexSettings.put(indexMetaData.getSettings());
                        }
                        metaDataBuilder.put(IndexMetaData.builder(indexMetaData).settings(indexSettings));
                    }
                }
            }
            if (!closeIndices.isEmpty()) {
                for (Index index : closeIndices) {
                    IndexMetaData indexMetaData = metaDataBuilder.getSafe(index);
                    Settings.Builder updates = Settings.builder();
                    Settings.Builder indexSettings = Settings.builder().put(indexMetaData.getSettings());
                    if (indexScopedSettings.updateSettings(closedSettings, indexSettings, updates, index.getName())) {
                        if (preserveExisting) {
                            indexSettings.put(indexMetaData.getSettings());
                        }
                        metaDataBuilder.put(IndexMetaData.builder(indexMetaData).settings(indexSettings));
                    }
                }
            }
            ClusterState updatedState = ClusterState.builder(currentState).metaData(metaDataBuilder).routingTable(routingTableBuilder.build()).blocks(blocks).build();
            // now, reroute in case things change that require it (like number of replicas)
            updatedState = allocationService.reroute(updatedState, "settings update");
            try {
                for (Index index : openIndices) {
                    final IndexMetaData currentMetaData = currentState.getMetaData().getIndexSafe(index);
                    final IndexMetaData updatedMetaData = updatedState.metaData().getIndexSafe(index);
                    indicesService.verifyIndexMetadata(currentMetaData, updatedMetaData);
                }
                for (Index index : closeIndices) {
                    final IndexMetaData currentMetaData = currentState.getMetaData().getIndexSafe(index);
                    final IndexMetaData updatedMetaData = updatedState.metaData().getIndexSafe(index);
                    indicesService.verifyIndexMetadata(currentMetaData, updatedMetaData);
                }
            } catch (IOException ex) {
                throw ExceptionsHelper.convertToElastic(ex);
            }
            return updatedState;
        }
    });
}

Example 42 with Index

use of org.elasticsearch.index.Index in project elasticsearch by elastic.

the class IndexRoutingTable method readFrom.

public static IndexRoutingTable readFrom(StreamInput in) throws IOException {
    Index index = new Index(in);
    Builder builder = new Builder(index);
    int size = in.readVInt();
    for (int i = 0; i < size; i++) {
        builder.addIndexShard(IndexShardRoutingTable.Builder.readFromThin(in, index));
    }
    return builder.build();
}

Example 43 with Index

use of org.elasticsearch.index.Index in project elasticsearch by elastic.

the class RoutingNodes method assertShardStats.

/**
     * Calculates RoutingNodes statistics by iterating over all {@link ShardRouting}s
     * in the cluster to ensure the book-keeping is correct.
     * For performance reasons, this should only be called from asserts
     *
     * @return this method always returns <code>true</code> or throws an assertion error. If assertion are not enabled
     *         this method does nothing.
     */
public static boolean assertShardStats(RoutingNodes routingNodes) {
    boolean run = false;
    // only run if assertions are enabled!
    assert (run = true);
    if (!run) {
        return true;
    }
    int unassignedPrimaryCount = 0;
    int unassignedIgnoredPrimaryCount = 0;
    int inactivePrimaryCount = 0;
    int inactiveShardCount = 0;
    int relocating = 0;
    Map<Index, Integer> indicesAndShards = new HashMap<>();
    for (RoutingNode node : routingNodes) {
        for (ShardRouting shard : node) {
            if (shard.initializing() && shard.relocatingNodeId() == null) {
                inactiveShardCount++;
                if (shard.primary()) {
                    inactivePrimaryCount++;
                }
            }
            if (shard.relocating()) {
                relocating++;
            }
            Integer i = indicesAndShards.get(shard.index());
            if (i == null) {
                i = shard.id();
            }
            indicesAndShards.put(shard.index(), Math.max(i, shard.id()));
        }
    }
    // Assert that the active shard routing are identical.
    Set<Map.Entry<Index, Integer>> entries = indicesAndShards.entrySet();
    final List<ShardRouting> shards = new ArrayList<>();
    for (Map.Entry<Index, Integer> e : entries) {
        Index index = e.getKey();
        for (int i = 0; i < e.getValue(); i++) {
            for (RoutingNode routingNode : routingNodes) {
                for (ShardRouting shardRouting : routingNode) {
                    if (shardRouting.index().equals(index) && shardRouting.id() == i) {
                        shards.add(shardRouting);
                    }
                }
            }
            List<ShardRouting> mutableShardRoutings = routingNodes.assignedShards(new ShardId(index, i));
            assert mutableShardRoutings.size() == shards.size();
            for (ShardRouting r : mutableShardRoutings) {
                assert shards.contains(r);
                shards.remove(r);
            }
            assert shards.isEmpty();
        }
    }
    for (ShardRouting shard : routingNodes.unassigned()) {
        if (shard.primary()) {
            unassignedPrimaryCount++;
        }
    }
    for (ShardRouting shard : routingNodes.unassigned().ignored()) {
        if (shard.primary()) {
            unassignedIgnoredPrimaryCount++;
        }
    }
    for (Map.Entry<String, Recoveries> recoveries : routingNodes.recoveriesPerNode.entrySet()) {
        String node = recoveries.getKey();
        final Recoveries value = recoveries.getValue();
        int incoming = 0;
        int outgoing = 0;
        RoutingNode routingNode = routingNodes.nodesToShards.get(node);
        if (routingNode != null) {
            // node might have dropped out of the cluster
            for (ShardRouting routing : routingNode) {
                if (routing.initializing()) {
                    incoming++;
                }
                if (routing.primary() && routing.isRelocationTarget() == false) {
                    for (ShardRouting assigned : routingNodes.assignedShards.get(routing.shardId())) {
                        if (assigned.initializing() && assigned.recoverySource().getType() == RecoverySource.Type.PEER) {
                            outgoing++;
                        }
                    }
                }
            }
        }
        assert incoming == value.incoming : incoming + " != " + value.incoming + " node: " + routingNode;
        assert outgoing == value.outgoing : outgoing + " != " + value.outgoing + " node: " + routingNode;
    }
    assert unassignedPrimaryCount == routingNodes.unassignedShards.getNumPrimaries() : "Unassigned primaries is [" + unassignedPrimaryCount + "] but RoutingNodes returned unassigned primaries [" + routingNodes.unassigned().getNumPrimaries() + "]";
    assert unassignedIgnoredPrimaryCount == routingNodes.unassignedShards.getNumIgnoredPrimaries() : "Unassigned ignored primaries is [" + unassignedIgnoredPrimaryCount + "] but RoutingNodes returned unassigned ignored primaries [" + routingNodes.unassigned().getNumIgnoredPrimaries() + "]";
    assert inactivePrimaryCount == routingNodes.inactivePrimaryCount : "Inactive Primary count [" + inactivePrimaryCount + "] but RoutingNodes returned inactive primaries [" + routingNodes.inactivePrimaryCount + "]";
    assert inactiveShardCount == routingNodes.inactiveShardCount : "Inactive Shard count [" + inactiveShardCount + "] but RoutingNodes returned inactive shards [" + routingNodes.inactiveShardCount + "]";
    assert routingNodes.getRelocatingShardCount() == relocating : "Relocating shards mismatch [" + routingNodes.getRelocatingShardCount() + "] but expected [" + relocating + "]";
    return true;
}

Example 44 with Index

use of org.elasticsearch.index.Index in project elasticsearch by elastic.

the class ClusterChangedEvent method indicesDeletedFromClusterState.

// Get the deleted indices by comparing the index metadatas in the previous and new cluster states.
// If an index exists in the previous cluster state, but not in the new cluster state, it must have been deleted.
private List<Index> indicesDeletedFromClusterState() {
    // https://github.com/elastic/elasticsearch/issues/11665
    if (metaDataChanged() == false || isNewCluster()) {
        return Collections.emptyList();
    }
    List<Index> deleted = null;
    for (ObjectCursor<IndexMetaData> cursor : previousState.metaData().indices().values()) {
        IndexMetaData index = cursor.value;
        IndexMetaData current = state.metaData().index(index.getIndex());
        if (current == null) {
            if (deleted == null) {
                deleted = new ArrayList<>();
            }
            deleted.add(index.getIndex());
        }
    }
    return deleted == null ? Collections.<Index>emptyList() : deleted;
}

Example 45 with Index

use of org.elasticsearch.index.Index in project elasticsearch by elastic.

the class Gateway method performStateRecovery.

public void performStateRecovery(final GatewayStateRecoveredListener listener) throws GatewayException {
    String[] nodesIds = clusterService.state().nodes().getMasterNodes().keys().toArray(String.class);
    logger.trace("performing state recovery from {}", Arrays.toString(nodesIds));
    TransportNodesListGatewayMetaState.NodesGatewayMetaState nodesState = listGatewayMetaState.list(nodesIds, null).actionGet();
    int requiredAllocation = Math.max(1, minimumMasterNodesProvider.get());
    if (nodesState.hasFailures()) {
        for (FailedNodeException failedNodeException : nodesState.failures()) {
            logger.warn("failed to fetch state from node", failedNodeException);
        }
    }
    ObjectFloatHashMap<Index> indices = new ObjectFloatHashMap<>();
    MetaData electedGlobalState = null;
    int found = 0;
    for (TransportNodesListGatewayMetaState.NodeGatewayMetaState nodeState : nodesState.getNodes()) {
        if (nodeState.metaData() == null) {
            continue;
        }
        found++;
        if (electedGlobalState == null) {
            electedGlobalState = nodeState.metaData();
        } else if (nodeState.metaData().version() > electedGlobalState.version()) {
            electedGlobalState = nodeState.metaData();
        }
        for (ObjectCursor<IndexMetaData> cursor : nodeState.metaData().indices().values()) {
            indices.addTo(cursor.value.getIndex(), 1);
        }
    }
    if (found < requiredAllocation) {
        listener.onFailure("found [" + found + "] metadata states, required [" + requiredAllocation + "]");
        return;
    }
    // update the global state, and clean the indices, we elect them in the next phase
    MetaData.Builder metaDataBuilder = MetaData.builder(electedGlobalState).removeAllIndices();
    assert !indices.containsKey(null);
    final Object[] keys = indices.keys;
    for (int i = 0; i < keys.length; i++) {
        if (keys[i] != null) {
            Index index = (Index) keys[i];
            IndexMetaData electedIndexMetaData = null;
            int indexMetaDataCount = 0;
            for (TransportNodesListGatewayMetaState.NodeGatewayMetaState nodeState : nodesState.getNodes()) {
                if (nodeState.metaData() == null) {
                    continue;
                }
                IndexMetaData indexMetaData = nodeState.metaData().index(index);
                if (indexMetaData == null) {
                    continue;
                }
                if (electedIndexMetaData == null) {
                    electedIndexMetaData = indexMetaData;
                } else if (indexMetaData.getVersion() > electedIndexMetaData.getVersion()) {
                    electedIndexMetaData = indexMetaData;
                }
                indexMetaDataCount++;
            }
            if (electedIndexMetaData != null) {
                if (indexMetaDataCount < requiredAllocation) {
                    logger.debug("[{}] found [{}], required [{}], not adding", index, indexMetaDataCount, requiredAllocation);
                }
                // TODO if this logging statement is correct then we are missing an else here
                try {
                    if (electedIndexMetaData.getState() == IndexMetaData.State.OPEN) {
                        // verify that we can actually create this index - if not we recover it as closed with lots of warn logs
                        indicesService.verifyIndexMetadata(electedIndexMetaData, electedIndexMetaData);
                    }
                } catch (Exception e) {
                    final Index electedIndex = electedIndexMetaData.getIndex();
                    logger.warn((org.apache.logging.log4j.util.Supplier<?>) () -> new ParameterizedMessage("recovering index {} failed - recovering as closed", electedIndex), e);
                    electedIndexMetaData = IndexMetaData.builder(electedIndexMetaData).state(IndexMetaData.State.CLOSE).build();
                }
                metaDataBuilder.put(electedIndexMetaData, false);
            }
        }
    }
    final ClusterSettings clusterSettings = clusterService.getClusterSettings();
    metaDataBuilder.persistentSettings(clusterSettings.archiveUnknownOrInvalidSettings(metaDataBuilder.persistentSettings(), e -> logUnknownSetting("persistent", e), (e, ex) -> logInvalidSetting("persistent", e, ex)));
    metaDataBuilder.transientSettings(clusterSettings.archiveUnknownOrInvalidSettings(metaDataBuilder.transientSettings(), e -> logUnknownSetting("transient", e), (e, ex) -> logInvalidSetting("transient", e, ex)));
    ClusterState.Builder builder = ClusterState.builder(clusterService.getClusterName());
    builder.metaData(metaDataBuilder);
    listener.onSuccess(builder.build());
}