Examples with Metadata org.elasticsearch.cluster.metadata.Metadata used on opensource projects

Example 41 with Metadata

use of org.elasticsearch.cluster.metadata.Metadata in project elasticsearch by elastic.

the class IndexMetaDataUpdater method removeStaleIdsWithoutRoutings.

/**
     * Removes allocation ids from the in-sync set for shard copies for which there is no routing entries in the routing table.
     * This method is called in AllocationService before any changes to the routing table are made.
     */
public static ClusterState removeStaleIdsWithoutRoutings(ClusterState clusterState, List<StaleShard> staleShards) {
    MetaData oldMetaData = clusterState.metaData();
    RoutingTable oldRoutingTable = clusterState.routingTable();
    MetaData.Builder metaDataBuilder = null;
    // group staleShards entries by index
    for (Map.Entry<Index, List<StaleShard>> indexEntry : staleShards.stream().collect(Collectors.groupingBy(fs -> fs.getShardId().getIndex())).entrySet()) {
        final IndexMetaData oldIndexMetaData = oldMetaData.getIndexSafe(indexEntry.getKey());
        IndexMetaData.Builder indexMetaDataBuilder = null;
        // group staleShards entries by shard id
        for (Map.Entry<ShardId, List<StaleShard>> shardEntry : indexEntry.getValue().stream().collect(Collectors.groupingBy(staleShard -> staleShard.getShardId())).entrySet()) {
            int shardNumber = shardEntry.getKey().getId();
            Set<String> oldInSyncAllocations = oldIndexMetaData.inSyncAllocationIds(shardNumber);
            Set<String> idsToRemove = shardEntry.getValue().stream().map(e -> e.getAllocationId()).collect(Collectors.toSet());
            assert idsToRemove.stream().allMatch(id -> oldRoutingTable.getByAllocationId(shardEntry.getKey(), id) == null) : "removing stale ids: " + idsToRemove + ", some of which have still a routing entry: " + oldRoutingTable;
            Set<String> remainingInSyncAllocations = Sets.difference(oldInSyncAllocations, idsToRemove);
            assert remainingInSyncAllocations.isEmpty() == false : "Set of in-sync ids cannot become empty for shard " + shardEntry.getKey() + " (before: " + oldInSyncAllocations + ", ids to remove: " + idsToRemove + ")";
            // (see ShardRouting#allocatedPostIndexCreate)
            if (remainingInSyncAllocations.isEmpty() == false) {
                if (indexMetaDataBuilder == null) {
                    indexMetaDataBuilder = IndexMetaData.builder(oldIndexMetaData);
                }
                indexMetaDataBuilder.putInSyncAllocationIds(shardNumber, remainingInSyncAllocations);
            }
        }
        if (indexMetaDataBuilder != null) {
            if (metaDataBuilder == null) {
                metaDataBuilder = MetaData.builder(oldMetaData);
            }
            metaDataBuilder.put(indexMetaDataBuilder);
        }
    }
    if (metaDataBuilder != null) {
        return ClusterState.builder(clusterState).metaData(metaDataBuilder).build();
    } else {
        return clusterState;
    }
}

Example 42 with Metadata

use of org.elasticsearch.cluster.metadata.Metadata in project elasticsearch by elastic.

the class IndexMetaDataUpdater method applyChanges.

/**
     * Updates the current {@link MetaData} based on the changes of this RoutingChangesObserver. Specifically
     * we update {@link IndexMetaData#getInSyncAllocationIds()} and {@link IndexMetaData#primaryTerm(int)} based on
     * the changes made during this allocation.
     *
     * @param oldMetaData {@link MetaData} object from before the routing nodes was changed.
     * @param newRoutingTable {@link RoutingTable} object after routing changes were applied.
     * @return adapted {@link MetaData}, potentially the original one if no change was needed.
     */
public MetaData applyChanges(MetaData oldMetaData, RoutingTable newRoutingTable) {
    Map<Index, List<Map.Entry<ShardId, Updates>>> changesGroupedByIndex = shardChanges.entrySet().stream().collect(Collectors.groupingBy(e -> e.getKey().getIndex()));
    MetaData.Builder metaDataBuilder = null;
    for (Map.Entry<Index, List<Map.Entry<ShardId, Updates>>> indexChanges : changesGroupedByIndex.entrySet()) {
        Index index = indexChanges.getKey();
        final IndexMetaData oldIndexMetaData = oldMetaData.getIndexSafe(index);
        IndexMetaData.Builder indexMetaDataBuilder = null;
        for (Map.Entry<ShardId, Updates> shardEntry : indexChanges.getValue()) {
            ShardId shardId = shardEntry.getKey();
            Updates updates = shardEntry.getValue();
            indexMetaDataBuilder = updateInSyncAllocations(newRoutingTable, oldIndexMetaData, indexMetaDataBuilder, shardId, updates);
            indexMetaDataBuilder = updatePrimaryTerm(oldIndexMetaData, indexMetaDataBuilder, shardId, updates);
        }
        if (indexMetaDataBuilder != null) {
            if (metaDataBuilder == null) {
                metaDataBuilder = MetaData.builder(oldMetaData);
            }
            metaDataBuilder.put(indexMetaDataBuilder);
        }
    }
    if (metaDataBuilder != null) {
        return metaDataBuilder.build();
    } else {
        return oldMetaData;
    }
}

Example 43 with Metadata

use of org.elasticsearch.cluster.metadata.Metadata in project elasticsearch by elastic.

the class AllocationService method buildResultAndLogHealthChange.

protected ClusterState buildResultAndLogHealthChange(ClusterState oldState, RoutingAllocation allocation, String reason) {
    RoutingTable oldRoutingTable = oldState.routingTable();
    RoutingNodes newRoutingNodes = allocation.routingNodes();
    final RoutingTable newRoutingTable = new RoutingTable.Builder().updateNodes(oldRoutingTable.version(), newRoutingNodes).build();
    MetaData newMetaData = allocation.updateMetaDataWithRoutingChanges(newRoutingTable);
    // validates the routing table is coherent with the cluster state metadata
    assert newRoutingTable.validate(newMetaData);
    final ClusterState.Builder newStateBuilder = ClusterState.builder(oldState).routingTable(newRoutingTable).metaData(newMetaData);
    final RestoreInProgress restoreInProgress = allocation.custom(RestoreInProgress.TYPE);
    if (restoreInProgress != null) {
        RestoreInProgress updatedRestoreInProgress = allocation.updateRestoreInfoWithRoutingChanges(restoreInProgress);
        if (updatedRestoreInProgress != restoreInProgress) {
            ImmutableOpenMap.Builder<String, ClusterState.Custom> customsBuilder = ImmutableOpenMap.builder(allocation.getCustoms());
            customsBuilder.put(RestoreInProgress.TYPE, updatedRestoreInProgress);
            newStateBuilder.customs(customsBuilder.build());
        }
    }
    final ClusterState newState = newStateBuilder.build();
    logClusterHealthStateChange(new ClusterStateHealth(oldState), new ClusterStateHealth(newState), reason);
    return newState;
}

Example 44 with Metadata

use of org.elasticsearch.cluster.metadata.Metadata in project elasticsearch by elastic.

the class Gateway method performStateRecovery.

public void performStateRecovery(final GatewayStateRecoveredListener listener) throws GatewayException {
    String[] nodesIds = clusterService.state().nodes().getMasterNodes().keys().toArray(String.class);
    logger.trace("performing state recovery from {}", Arrays.toString(nodesIds));
    TransportNodesListGatewayMetaState.NodesGatewayMetaState nodesState = listGatewayMetaState.list(nodesIds, null).actionGet();
    int requiredAllocation = Math.max(1, minimumMasterNodesProvider.get());
    if (nodesState.hasFailures()) {
        for (FailedNodeException failedNodeException : nodesState.failures()) {
            logger.warn("failed to fetch state from node", failedNodeException);
        }
    }
    ObjectFloatHashMap<Index> indices = new ObjectFloatHashMap<>();
    MetaData electedGlobalState = null;
    int found = 0;
    for (TransportNodesListGatewayMetaState.NodeGatewayMetaState nodeState : nodesState.getNodes()) {
        if (nodeState.metaData() == null) {
            continue;
        }
        found++;
        if (electedGlobalState == null) {
            electedGlobalState = nodeState.metaData();
        } else if (nodeState.metaData().version() > electedGlobalState.version()) {
            electedGlobalState = nodeState.metaData();
        }
        for (ObjectCursor<IndexMetaData> cursor : nodeState.metaData().indices().values()) {
            indices.addTo(cursor.value.getIndex(), 1);
        }
    }
    if (found < requiredAllocation) {
        listener.onFailure("found [" + found + "] metadata states, required [" + requiredAllocation + "]");
        return;
    }
    // update the global state, and clean the indices, we elect them in the next phase
    MetaData.Builder metaDataBuilder = MetaData.builder(electedGlobalState).removeAllIndices();
    assert !indices.containsKey(null);
    final Object[] keys = indices.keys;
    for (int i = 0; i < keys.length; i++) {
        if (keys[i] != null) {
            Index index = (Index) keys[i];
            IndexMetaData electedIndexMetaData = null;
            int indexMetaDataCount = 0;
            for (TransportNodesListGatewayMetaState.NodeGatewayMetaState nodeState : nodesState.getNodes()) {
                if (nodeState.metaData() == null) {
                    continue;
                }
                IndexMetaData indexMetaData = nodeState.metaData().index(index);
                if (indexMetaData == null) {
                    continue;
                }
                if (electedIndexMetaData == null) {
                    electedIndexMetaData = indexMetaData;
                } else if (indexMetaData.getVersion() > electedIndexMetaData.getVersion()) {
                    electedIndexMetaData = indexMetaData;
                }
                indexMetaDataCount++;
            }
            if (electedIndexMetaData != null) {
                if (indexMetaDataCount < requiredAllocation) {
                    logger.debug("[{}] found [{}], required [{}], not adding", index, indexMetaDataCount, requiredAllocation);
                }
                // TODO if this logging statement is correct then we are missing an else here
                try {
                    if (electedIndexMetaData.getState() == IndexMetaData.State.OPEN) {
                        // verify that we can actually create this index - if not we recover it as closed with lots of warn logs
                        indicesService.verifyIndexMetadata(electedIndexMetaData, electedIndexMetaData);
                    }
                } catch (Exception e) {
                    final Index electedIndex = electedIndexMetaData.getIndex();
                    logger.warn((org.apache.logging.log4j.util.Supplier<?>) () -> new ParameterizedMessage("recovering index {} failed - recovering as closed", electedIndex), e);
                    electedIndexMetaData = IndexMetaData.builder(electedIndexMetaData).state(IndexMetaData.State.CLOSE).build();
                }
                metaDataBuilder.put(electedIndexMetaData, false);
            }
        }
    }
    final ClusterSettings clusterSettings = clusterService.getClusterSettings();
    metaDataBuilder.persistentSettings(clusterSettings.archiveUnknownOrInvalidSettings(metaDataBuilder.persistentSettings(), e -> logUnknownSetting("persistent", e), (e, ex) -> logInvalidSetting("persistent", e, ex)));
    metaDataBuilder.transientSettings(clusterSettings.archiveUnknownOrInvalidSettings(metaDataBuilder.transientSettings(), e -> logUnknownSetting("transient", e), (e, ex) -> logInvalidSetting("transient", e, ex)));
    ClusterState.Builder builder = ClusterState.builder(clusterService.getClusterName());
    builder.metaData(metaDataBuilder);
    listener.onSuccess(builder.build());
}

Example 45 with Metadata

use of org.elasticsearch.cluster.metadata.Metadata in project elasticsearch by elastic.

the class GatewayMetaState method applyClusterState.

@Override
public void applyClusterState(ClusterChangedEvent event) {
    final ClusterState state = event.state();
    if (state.blocks().disableStatePersistence()) {
        // reset the current metadata, we need to start fresh...
        this.previousMetaData = null;
        previouslyWrittenIndices = emptySet();
        return;
    }
    MetaData newMetaData = state.metaData();
    // we don't check if metaData changed, since we might be called several times and we need to check dangling...
    Set<Index> relevantIndices = Collections.emptySet();
    boolean success = true;
    // write the state if this node is a master eligible node or if it is a data node and has shards allocated on it
    if (state.nodes().getLocalNode().isMasterNode() || state.nodes().getLocalNode().isDataNode()) {
        if (previousMetaData == null) {
            try {
                // we therefore have to check here if we have shards on disk and add their indices to the previouslyWrittenIndices list
                if (isDataOnlyNode(state)) {
                    Set<Index> newPreviouslyWrittenIndices = new HashSet<>(previouslyWrittenIndices.size());
                    for (IndexMetaData indexMetaData : newMetaData) {
                        IndexMetaData indexMetaDataOnDisk = null;
                        if (indexMetaData.getState().equals(IndexMetaData.State.CLOSE)) {
                            indexMetaDataOnDisk = metaStateService.loadIndexState(indexMetaData.getIndex());
                        }
                        if (indexMetaDataOnDisk != null) {
                            newPreviouslyWrittenIndices.add(indexMetaDataOnDisk.getIndex());
                        }
                    }
                    newPreviouslyWrittenIndices.addAll(previouslyWrittenIndices);
                    previouslyWrittenIndices = unmodifiableSet(newPreviouslyWrittenIndices);
                }
            } catch (Exception e) {
                success = false;
            }
        }
        // check if the global state changed?
        if (previousMetaData == null || !MetaData.isGlobalStateEquals(previousMetaData, newMetaData)) {
            try {
                metaStateService.writeGlobalState("changed", newMetaData);
            } catch (Exception e) {
                success = false;
            }
        }
        relevantIndices = getRelevantIndices(event.state(), event.previousState(), previouslyWrittenIndices);
        final Iterable<IndexMetaWriteInfo> writeInfo = resolveStatesToBeWritten(previouslyWrittenIndices, relevantIndices, previousMetaData, event.state().metaData());
        // check and write changes in indices
        for (IndexMetaWriteInfo indexMetaWrite : writeInfo) {
            try {
                metaStateService.writeIndex(indexMetaWrite.reason, indexMetaWrite.newMetaData);
            } catch (Exception e) {
                success = false;
            }
        }
    }
    if (success) {
        previousMetaData = newMetaData;
        previouslyWrittenIndices = unmodifiableSet(relevantIndices);
    }
}