Search in sources :

Example 1 with MetaDataUpdateSettingsService

use of org.elasticsearch.cluster.metadata.MetaDataUpdateSettingsService in project crate by crate.

the class AlterTableClusterStateExecutor method updateSettings.

/**
 * The logic is taken over from {@link MetadataUpdateSettingsService#updateSettings(UpdateSettingsClusterStateUpdateRequest, ActionListener)}
 */
private ClusterState updateSettings(final ClusterState currentState, final Settings settings, Index[] concreteIndices) {
    final Settings normalizedSettings = Settings.builder().put(markArchivedSettings(settings)).normalizePrefix(IndexMetadata.INDEX_SETTING_PREFIX).build();
    Settings.Builder settingsForClosedIndices = Settings.builder();
    Settings.Builder settingsForOpenIndices = Settings.builder();
    final Set<String> skippedSettings = new HashSet<>();
    for (String key : normalizedSettings.keySet()) {
        Setting setting = indexScopedSettings.get(key);
        boolean isWildcard = setting == null && Regex.isSimpleMatchPattern(key);
        assert // we already validated the normalized settings
        setting != null || (isWildcard && normalizedSettings.hasValue(key) == false) : "unknown setting: " + key + " isWildcard: " + isWildcard + " hasValue: " + normalizedSettings.hasValue(key);
        settingsForClosedIndices.copy(key, normalizedSettings);
        if (isWildcard || setting.isDynamic()) {
            settingsForOpenIndices.copy(key, normalizedSettings);
        } else {
            skippedSettings.add(key);
        }
    }
    final Settings closedSettings = settingsForClosedIndices.build();
    final Settings openSettings = settingsForOpenIndices.build();
    final RoutingTable.Builder routingTableBuilder = RoutingTable.builder(currentState.routingTable());
    final Metadata.Builder metadataBuilder = Metadata.builder(currentState.metadata());
    // allow to change any settings to a close index, and only allow dynamic settings to be changed
    // on an open index
    Set<Index> openIndices = new HashSet<>();
    Set<Index> closeIndices = new HashSet<>();
    final String[] actualIndices = new String[concreteIndices.length];
    for (int i = 0; i < concreteIndices.length; i++) {
        Index index = concreteIndices[i];
        actualIndices[i] = index.getName();
        final IndexMetadata metadata = currentState.metadata().getIndexSafe(index);
        if (metadata.getState() == IndexMetadata.State.OPEN) {
            openIndices.add(index);
        } else {
            closeIndices.add(index);
        }
    }
    if (!skippedSettings.isEmpty() && !openIndices.isEmpty()) {
        throw new IllegalArgumentException(String.format(Locale.ROOT, "Can't update non dynamic settings [%s] for open indices %s", skippedSettings, openIndices));
    }
    if (IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.exists(openSettings)) {
        final int updatedNumberOfReplicas = IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.get(openSettings);
        // Verify that this won't take us over the cluster shard limit.
        int totalNewShards = openIndices.stream().mapToInt(i -> MetadataUpdateSettingsService.getTotalNewShards(i, currentState, updatedNumberOfReplicas)).sum();
        Optional<String> error = shardLimitValidator.checkShardLimit(totalNewShards, currentState);
        if (error.isPresent()) {
            ValidationException ex = new ValidationException();
            ex.addValidationError(error.get());
            throw ex;
        }
        /*
             * We do not update the in-sync allocation IDs as they will be removed upon the first index operation which makes
             * these copies stale.
             *
             * TODO: should we update the in-sync allocation IDs once the data is deleted by the node?
             */
        routingTableBuilder.updateNumberOfReplicas(updatedNumberOfReplicas, actualIndices);
        metadataBuilder.updateNumberOfReplicas(updatedNumberOfReplicas, actualIndices);
    }
    ClusterBlocks.Builder blocks = ClusterBlocks.builder().blocks(currentState.blocks());
    maybeUpdateClusterBlock(actualIndices, blocks, IndexMetadata.INDEX_READ_ONLY_BLOCK, IndexMetadata.INDEX_READ_ONLY_SETTING, openSettings);
    maybeUpdateClusterBlock(actualIndices, blocks, IndexMetadata.INDEX_READ_ONLY_ALLOW_DELETE_BLOCK, IndexMetadata.INDEX_BLOCKS_READ_ONLY_ALLOW_DELETE_SETTING, openSettings);
    maybeUpdateClusterBlock(actualIndices, blocks, IndexMetadata.INDEX_METADATA_BLOCK, IndexMetadata.INDEX_BLOCKS_METADATA_SETTING, openSettings);
    maybeUpdateClusterBlock(actualIndices, blocks, IndexMetadata.INDEX_WRITE_BLOCK, IndexMetadata.INDEX_BLOCKS_WRITE_SETTING, openSettings);
    maybeUpdateClusterBlock(actualIndices, blocks, IndexMetadata.INDEX_READ_BLOCK, IndexMetadata.INDEX_BLOCKS_READ_SETTING, openSettings);
    if (!openIndices.isEmpty()) {
        for (Index index : openIndices) {
            IndexMetadata indexMetadata = metadataBuilder.getSafe(index);
            Settings.Builder updates = Settings.builder();
            Settings.Builder indexSettings = Settings.builder().put(indexMetadata.getSettings());
            if (indexScopedSettings.updateDynamicSettings(openSettings, indexSettings, updates, index.getName())) {
                Settings finalSettings = indexSettings.build();
                indexScopedSettings.validate(finalSettings.filter(k -> indexScopedSettings.isPrivateSetting(k) == false), true);
                metadataBuilder.put(IndexMetadata.builder(indexMetadata).settings(finalSettings));
            }
        }
    }
    if (!closeIndices.isEmpty()) {
        for (Index index : closeIndices) {
            IndexMetadata indexMetadata = metadataBuilder.getSafe(index);
            Settings.Builder updates = Settings.builder();
            Settings.Builder indexSettings = Settings.builder().put(indexMetadata.getSettings());
            if (indexScopedSettings.updateSettings(closedSettings, indexSettings, updates, index.getName())) {
                Settings finalSettings = indexSettings.build();
                indexScopedSettings.validate(finalSettings.filter(k -> indexScopedSettings.isPrivateSetting(k) == false), true);
                metadataBuilder.put(IndexMetadata.builder(indexMetadata).settings(finalSettings));
            }
        }
    }
    // increment settings versions
    for (final String index : actualIndices) {
        if (same(currentState.metadata().index(index).getSettings(), metadataBuilder.get(index).getSettings()) == false) {
            final IndexMetadata.Builder builder = IndexMetadata.builder(metadataBuilder.get(index));
            builder.settingsVersion(1 + builder.settingsVersion());
            metadataBuilder.put(builder);
        }
    }
    ClusterState updatedState = ClusterState.builder(currentState).metadata(metadataBuilder).routingTable(routingTableBuilder.build()).blocks(blocks).build();
    // now, reroute in case things change that require it (like number of replicas)
    updatedState = allocationService.reroute(updatedState, "settings update");
    try {
        for (Index index : openIndices) {
            final IndexMetadata currentMetadata = currentState.getMetadata().getIndexSafe(index);
            final IndexMetadata updatedMetadata = updatedState.metadata().getIndexSafe(index);
            indicesService.verifyIndexMetadata(currentMetadata, updatedMetadata);
        }
        for (Index index : closeIndices) {
            final IndexMetadata currentMetadata = currentState.getMetadata().getIndexSafe(index);
            final IndexMetadata updatedMetadata = updatedState.metadata().getIndexSafe(index);
            // Verifies that the current index settings can be updated with the updated dynamic settings.
            indicesService.verifyIndexMetadata(currentMetadata, updatedMetadata);
            // Now check that we can create the index with the updated settings (dynamic and non-dynamic).
            // This step is mandatory since we allow to update non-dynamic settings on closed indices.
            indicesService.verifyIndexMetadata(updatedMetadata, updatedMetadata);
        }
    } catch (IOException ex) {
        throw new UncheckedIOException(ex);
    }
    return updatedState;
}
Also used : ShardLimitValidator(org.elasticsearch.indices.ShardLimitValidator) RelationName(io.crate.metadata.RelationName) IndexMetadata(org.elasticsearch.cluster.metadata.IndexMetadata) AllocationService(org.elasticsearch.cluster.routing.allocation.AllocationService) ClusterBlocks(org.elasticsearch.cluster.block.ClusterBlocks) ClusterState(org.elasticsearch.cluster.ClusterState) Settings(org.elasticsearch.common.settings.Settings) Locale(java.util.Locale) Map(java.util.Map) IndicesOptions(org.elasticsearch.action.support.IndicesOptions) ValidationException(org.elasticsearch.common.ValidationException) MetadataUpdateSettingsService.maybeUpdateClusterBlock(org.elasticsearch.cluster.metadata.MetadataUpdateSettingsService.maybeUpdateClusterBlock) InvalidIndexTemplateException(org.elasticsearch.indices.InvalidIndexTemplateException) NodeContext(io.crate.metadata.NodeContext) IndexSettings.same(org.elasticsearch.index.IndexSettings.same) Setting(org.elasticsearch.common.settings.Setting) Set(java.util.Set) BytesReference(org.elasticsearch.common.bytes.BytesReference) Collectors(java.util.stream.Collectors) TableParameters(io.crate.analyze.TableParameters) UncheckedIOException(java.io.UncheckedIOException) MapperService(org.elasticsearch.index.mapper.MapperService) List(java.util.List) Optional(java.util.Optional) PutMappingClusterStateUpdateRequest(org.elasticsearch.action.admin.indices.mapping.put.PutMappingClusterStateUpdateRequest) VisibleForTesting(io.crate.common.annotations.VisibleForTesting) DocTableInfoBuilder(io.crate.metadata.doc.DocTableInfoBuilder) XContentFactory(org.elasticsearch.common.xcontent.XContentFactory) XContentType(org.elasticsearch.common.xcontent.XContentType) AlterTableRequest(io.crate.execution.ddl.tables.AlterTableRequest) HashMap(java.util.HashMap) Index(org.elasticsearch.index.Index) MetadataUpdateSettingsService(org.elasticsearch.cluster.metadata.MetadataUpdateSettingsService) PartitionName(io.crate.metadata.PartitionName) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) XContentHelper(org.elasticsearch.common.xcontent.XContentHelper) Metadata(org.elasticsearch.cluster.metadata.Metadata) BiConsumer(java.util.function.BiConsumer) Regex(org.elasticsearch.common.regex.Regex) IndicesService(org.elasticsearch.indices.IndicesService) MetadataMappingService(org.elasticsearch.cluster.metadata.MetadataMappingService) Maps(io.crate.common.collections.Maps) IOException(java.io.IOException) IndexTemplateMetadata(org.elasticsearch.cluster.metadata.IndexTemplateMetadata) MetadataCreateIndexService(org.elasticsearch.cluster.metadata.MetadataCreateIndexService) IndexScopedSettings(org.elasticsearch.common.settings.IndexScopedSettings) RoutingTable(org.elasticsearch.cluster.routing.RoutingTable) UpdateSettingsClusterStateUpdateRequest(org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsClusterStateUpdateRequest) IndexNameExpressionResolver(org.elasticsearch.cluster.metadata.IndexNameExpressionResolver) ARCHIVED_SETTINGS_PREFIX(org.elasticsearch.common.settings.AbstractScopedSettings.ARCHIVED_SETTINGS_PREFIX) Collections(java.util.Collections) ActionListener(org.elasticsearch.action.ActionListener) ValidationException(org.elasticsearch.common.ValidationException) IndexMetadata(org.elasticsearch.cluster.metadata.IndexMetadata) Metadata(org.elasticsearch.cluster.metadata.Metadata) IndexTemplateMetadata(org.elasticsearch.cluster.metadata.IndexTemplateMetadata) Index(org.elasticsearch.index.Index) UncheckedIOException(java.io.UncheckedIOException) IndexMetadata(org.elasticsearch.cluster.metadata.IndexMetadata) Settings(org.elasticsearch.common.settings.Settings) IndexScopedSettings(org.elasticsearch.common.settings.IndexScopedSettings) HashSet(java.util.HashSet) ClusterState(org.elasticsearch.cluster.ClusterState) ClusterBlocks(org.elasticsearch.cluster.block.ClusterBlocks) Setting(org.elasticsearch.common.settings.Setting) UncheckedIOException(java.io.UncheckedIOException) IOException(java.io.IOException) RoutingTable(org.elasticsearch.cluster.routing.RoutingTable)

Aggregations

TableParameters (io.crate.analyze.TableParameters)1 VisibleForTesting (io.crate.common.annotations.VisibleForTesting)1 Maps (io.crate.common.collections.Maps)1 AlterTableRequest (io.crate.execution.ddl.tables.AlterTableRequest)1 NodeContext (io.crate.metadata.NodeContext)1 PartitionName (io.crate.metadata.PartitionName)1 RelationName (io.crate.metadata.RelationName)1 DocTableInfoBuilder (io.crate.metadata.doc.DocTableInfoBuilder)1 IOException (java.io.IOException)1 UncheckedIOException (java.io.UncheckedIOException)1 ArrayList (java.util.ArrayList)1 Collections (java.util.Collections)1 HashMap (java.util.HashMap)1 HashSet (java.util.HashSet)1 List (java.util.List)1 Locale (java.util.Locale)1 Map (java.util.Map)1 Optional (java.util.Optional)1 Set (java.util.Set)1 BiConsumer (java.util.function.BiConsumer)1