Examples with IndexMetadata org.elasticsearch.cluster.metadata.IndexMetadata used on opensource projects

Example 36 with IndexMetadata

use of org.elasticsearch.cluster.metadata.IndexMetadata in project elasticsearch by elastic.

the class TransportRolloverAction method masterOperation.

@Override
protected void masterOperation(final RolloverRequest rolloverRequest, final ClusterState state, final ActionListener<RolloverResponse> listener) {
    final MetaData metaData = state.metaData();
    validate(metaData, rolloverRequest);
    final AliasOrIndex aliasOrIndex = metaData.getAliasAndIndexLookup().get(rolloverRequest.getAlias());
    final IndexMetaData indexMetaData = aliasOrIndex.getIndices().get(0);
    final String sourceProvidedName = indexMetaData.getSettings().get(IndexMetaData.SETTING_INDEX_PROVIDED_NAME, indexMetaData.getIndex().getName());
    final String sourceIndexName = indexMetaData.getIndex().getName();
    final String unresolvedName = (rolloverRequest.getNewIndexName() != null) ? rolloverRequest.getNewIndexName() : generateRolloverIndexName(sourceProvidedName, indexNameExpressionResolver);
    final String rolloverIndexName = indexNameExpressionResolver.resolveDateMathExpression(unresolvedName);
    // will fail if the index already exists
    MetaDataCreateIndexService.validateIndexName(rolloverIndexName, state);
    client.admin().indices().prepareStats(sourceIndexName).clear().setDocs(true).execute(new ActionListener<IndicesStatsResponse>() {

        @Override
        public void onResponse(IndicesStatsResponse statsResponse) {
            final Set<Condition.Result> conditionResults = evaluateConditions(rolloverRequest.getConditions(), statsResponse.getTotal().getDocs(), metaData.index(sourceIndexName));
            if (rolloverRequest.isDryRun()) {
                listener.onResponse(new RolloverResponse(sourceIndexName, rolloverIndexName, conditionResults, true, false, false, false));
                return;
            }
            if (conditionResults.size() == 0 || conditionResults.stream().anyMatch(result -> result.matched)) {
                CreateIndexClusterStateUpdateRequest updateRequest = prepareCreateIndexRequest(unresolvedName, rolloverIndexName, rolloverRequest);
                createIndexService.createIndex(updateRequest, ActionListener.wrap(createIndexClusterStateUpdateResponse -> {
                    indexAliasesService.indicesAliases(prepareRolloverAliasesUpdateRequest(sourceIndexName, rolloverIndexName, rolloverRequest), ActionListener.wrap(aliasClusterStateUpdateResponse -> {
                        if (aliasClusterStateUpdateResponse.isAcknowledged()) {
                            activeShardsObserver.waitForActiveShards(rolloverIndexName, rolloverRequest.getCreateIndexRequest().waitForActiveShards(), rolloverRequest.masterNodeTimeout(), isShardsAcked -> listener.onResponse(new RolloverResponse(sourceIndexName, rolloverIndexName, conditionResults, false, true, true, isShardsAcked)), listener::onFailure);
                        } else {
                            listener.onResponse(new RolloverResponse(sourceIndexName, rolloverIndexName, conditionResults, false, true, false, false));
                        }
                    }, listener::onFailure));
                }, listener::onFailure));
            } else {
                // conditions not met
                listener.onResponse(new RolloverResponse(sourceIndexName, rolloverIndexName, conditionResults, false, false, false, false));
            }
        }

        @Override
        public void onFailure(Exception e) {
            listener.onFailure(e);
        }
    });
}

Example 37 with IndexMetadata

use of org.elasticsearch.cluster.metadata.IndexMetadata in project elasticsearch by elastic.

the class TransportGetSettingsAction method masterOperation.

@Override
protected void masterOperation(GetSettingsRequest request, ClusterState state, ActionListener<GetSettingsResponse> listener) {
    Index[] concreteIndices = indexNameExpressionResolver.concreteIndices(state, request);
    ImmutableOpenMap.Builder<String, Settings> indexToSettingsBuilder = ImmutableOpenMap.builder();
    for (Index concreteIndex : concreteIndices) {
        IndexMetaData indexMetaData = state.getMetaData().index(concreteIndex);
        if (indexMetaData == null) {
            continue;
        }
        Settings settings = settingsFilter.filter(indexMetaData.getSettings());
        if (request.humanReadable()) {
            settings = IndexMetaData.addHumanReadableSettings(settings);
        }
        if (!CollectionUtils.isEmpty(request.names())) {
            Settings.Builder settingsBuilder = Settings.builder();
            for (Map.Entry<String, String> entry : settings.getAsMap().entrySet()) {
                if (Regex.simpleMatch(request.names(), entry.getKey())) {
                    settingsBuilder.put(entry.getKey(), entry.getValue());
                }
            }
            settings = settingsBuilder.build();
        }
        indexToSettingsBuilder.put(concreteIndex.getName(), settings);
    }
    listener.onResponse(new GetSettingsResponse(indexToSettingsBuilder.build()));
}

Example 38 with IndexMetadata

use of org.elasticsearch.cluster.metadata.IndexMetadata in project elasticsearch by elastic.

the class TransportShardBulkAction method shardOperationOnPrimary.

@Override
public WritePrimaryResult<BulkShardRequest, BulkShardResponse> shardOperationOnPrimary(BulkShardRequest request, IndexShard primary) throws Exception {
    final IndexMetaData metaData = primary.indexSettings().getIndexMetaData();
    Translog.Location location = null;
    final MappingUpdatePerformer mappingUpdater = new ConcreteMappingUpdatePerformer();
    for (int requestIndex = 0; requestIndex < request.items().length; requestIndex++) {
        location = executeBulkItemRequest(metaData, primary, request, location, requestIndex, updateHelper, threadPool::absoluteTimeInMillis, mappingUpdater);
    }
    BulkItemResponse[] responses = new BulkItemResponse[request.items().length];
    BulkItemRequest[] items = request.items();
    for (int i = 0; i < items.length; i++) {
        responses[i] = items[i].getPrimaryResponse();
    }
    BulkShardResponse response = new BulkShardResponse(request.shardId(), responses);
    return new WritePrimaryResult<>(request, response, location, null, primary, logger);
}

Example 39 with IndexMetadata

use of org.elasticsearch.cluster.metadata.IndexMetadata in project elasticsearch by elastic.

the class IndexMetaDataUpdater method updateInSyncAllocations.

/**
     * Updates in-sync allocations with routing changes that were made to the routing table.
     */
private IndexMetaData.Builder updateInSyncAllocations(RoutingTable newRoutingTable, IndexMetaData oldIndexMetaData, IndexMetaData.Builder indexMetaDataBuilder, ShardId shardId, Updates updates) {
    assert Sets.haveEmptyIntersection(updates.addedAllocationIds, updates.removedAllocationIds) : "allocation ids cannot be both added and removed in the same allocation round, added ids: " + updates.addedAllocationIds + ", removed ids: " + updates.removedAllocationIds;
    Set<String> oldInSyncAllocationIds = oldIndexMetaData.inSyncAllocationIds(shardId.id());
    // check if we have been force-initializing an empty primary or a stale primary
    if (updates.initializedPrimary != null && oldInSyncAllocationIds.isEmpty() == false && oldInSyncAllocationIds.contains(updates.initializedPrimary.allocationId().getId()) == false) {
        // we're not reusing an existing in-sync allocation id to initialize a primary, which means that we're either force-allocating
        // an empty or a stale primary (see AllocateEmptyPrimaryAllocationCommand or AllocateStalePrimaryAllocationCommand).
        RecoverySource.Type recoverySourceType = updates.initializedPrimary.recoverySource().getType();
        boolean emptyPrimary = recoverySourceType == RecoverySource.Type.EMPTY_STORE;
        assert updates.addedAllocationIds.isEmpty() : (emptyPrimary ? "empty" : "stale") + " primary is not force-initialized in same allocation round where shards are started";
        if (indexMetaDataBuilder == null) {
            indexMetaDataBuilder = IndexMetaData.builder(oldIndexMetaData);
        }
        if (emptyPrimary) {
            // forcing an empty primary resets the in-sync allocations to the empty set (ShardRouting.allocatedPostIndexCreate)
            indexMetaDataBuilder.putInSyncAllocationIds(shardId.id(), Collections.emptySet());
        } else {
            // forcing a stale primary resets the in-sync allocations to the singleton set with the stale id
            indexMetaDataBuilder.putInSyncAllocationIds(shardId.id(), Collections.singleton(updates.initializedPrimary.allocationId().getId()));
        }
    } else {
        // standard path for updating in-sync ids
        Set<String> inSyncAllocationIds = new HashSet<>(oldInSyncAllocationIds);
        inSyncAllocationIds.addAll(updates.addedAllocationIds);
        inSyncAllocationIds.removeAll(updates.removedAllocationIds);
        // Prevent set of inSyncAllocationIds to grow unboundedly. This can happen for example if we don't write to a primary
        // but repeatedly shut down nodes that have active replicas.
        // We use number_of_replicas + 1 (= possible active shard copies) to bound the inSyncAllocationIds set
        // Only trim the set of allocation ids when it grows, otherwise we might trim too eagerly when the number
        // of replicas was decreased while shards were unassigned.
        // +1 for the primary
        int maxActiveShards = oldIndexMetaData.getNumberOfReplicas() + 1;
        IndexShardRoutingTable newShardRoutingTable = newRoutingTable.shardRoutingTable(shardId);
        if (inSyncAllocationIds.size() > oldInSyncAllocationIds.size() && inSyncAllocationIds.size() > maxActiveShards) {
            // trim entries that have no corresponding shard routing in the cluster state (i.e. trim unavailable copies)
            List<ShardRouting> assignedShards = newShardRoutingTable.assignedShards();
            assert assignedShards.size() <= maxActiveShards : "cannot have more assigned shards " + assignedShards + " than maximum possible active shards " + maxActiveShards;
            Set<String> assignedAllocations = assignedShards.stream().map(s -> s.allocationId().getId()).collect(Collectors.toSet());
            inSyncAllocationIds = inSyncAllocationIds.stream().sorted(// values with routing entries first
            Comparator.comparing(assignedAllocations::contains).reversed()).limit(maxActiveShards).collect(Collectors.toSet());
        }
        // in-sync set, this could create an empty primary on the next allocation.
        if (newShardRoutingTable.activeShards().isEmpty() && updates.firstFailedPrimary != null) {
            // add back allocation id of failed primary
            inSyncAllocationIds.add(updates.firstFailedPrimary.allocationId().getId());
        }
        assert inSyncAllocationIds.isEmpty() == false || oldInSyncAllocationIds.isEmpty() : "in-sync allocations cannot become empty after they have been non-empty: " + oldInSyncAllocationIds;
        // be extra safe here and only update in-sync set if it is non-empty
        if (inSyncAllocationIds.isEmpty() == false) {
            if (indexMetaDataBuilder == null) {
                indexMetaDataBuilder = IndexMetaData.builder(oldIndexMetaData);
            }
            indexMetaDataBuilder.putInSyncAllocationIds(shardId.id(), inSyncAllocationIds);
        }
    }
    return indexMetaDataBuilder;
}

Example 40 with IndexMetadata

use of org.elasticsearch.cluster.metadata.IndexMetadata in project elasticsearch by elastic.

the class IndexMetaDataUpdater method removeStaleIdsWithoutRoutings.

/**
     * Removes allocation ids from the in-sync set for shard copies for which there is no routing entries in the routing table.
     * This method is called in AllocationService before any changes to the routing table are made.
     */
public static ClusterState removeStaleIdsWithoutRoutings(ClusterState clusterState, List<StaleShard> staleShards) {
    MetaData oldMetaData = clusterState.metaData();
    RoutingTable oldRoutingTable = clusterState.routingTable();
    MetaData.Builder metaDataBuilder = null;
    // group staleShards entries by index
    for (Map.Entry<Index, List<StaleShard>> indexEntry : staleShards.stream().collect(Collectors.groupingBy(fs -> fs.getShardId().getIndex())).entrySet()) {
        final IndexMetaData oldIndexMetaData = oldMetaData.getIndexSafe(indexEntry.getKey());
        IndexMetaData.Builder indexMetaDataBuilder = null;
        // group staleShards entries by shard id
        for (Map.Entry<ShardId, List<StaleShard>> shardEntry : indexEntry.getValue().stream().collect(Collectors.groupingBy(staleShard -> staleShard.getShardId())).entrySet()) {
            int shardNumber = shardEntry.getKey().getId();
            Set<String> oldInSyncAllocations = oldIndexMetaData.inSyncAllocationIds(shardNumber);
            Set<String> idsToRemove = shardEntry.getValue().stream().map(e -> e.getAllocationId()).collect(Collectors.toSet());
            assert idsToRemove.stream().allMatch(id -> oldRoutingTable.getByAllocationId(shardEntry.getKey(), id) == null) : "removing stale ids: " + idsToRemove + ", some of which have still a routing entry: " + oldRoutingTable;
            Set<String> remainingInSyncAllocations = Sets.difference(oldInSyncAllocations, idsToRemove);
            assert remainingInSyncAllocations.isEmpty() == false : "Set of in-sync ids cannot become empty for shard " + shardEntry.getKey() + " (before: " + oldInSyncAllocations + ", ids to remove: " + idsToRemove + ")";
            // (see ShardRouting#allocatedPostIndexCreate)
            if (remainingInSyncAllocations.isEmpty() == false) {
                if (indexMetaDataBuilder == null) {
                    indexMetaDataBuilder = IndexMetaData.builder(oldIndexMetaData);
                }
                indexMetaDataBuilder.putInSyncAllocationIds(shardNumber, remainingInSyncAllocations);
            }
        }
        if (indexMetaDataBuilder != null) {
            if (metaDataBuilder == null) {
                metaDataBuilder = MetaData.builder(oldMetaData);
            }
            metaDataBuilder.put(indexMetaDataBuilder);
        }
    }
    if (metaDataBuilder != null) {
        return ClusterState.builder(clusterState).metaData(metaDataBuilder).build();
    } else {
        return clusterState;
    }
}