Examples with IndexMetaData org.elasticsearch.cluster.metadata.IndexMetaData used on opensource projects

Example 46 with IndexMetaData

use of org.elasticsearch.cluster.metadata.IndexMetaData in project elasticsearch by elastic.

the class Gateway method performStateRecovery.

public void performStateRecovery(final GatewayStateRecoveredListener listener) throws GatewayException {
    String[] nodesIds = clusterService.state().nodes().getMasterNodes().keys().toArray(String.class);
    logger.trace("performing state recovery from {}", Arrays.toString(nodesIds));
    TransportNodesListGatewayMetaState.NodesGatewayMetaState nodesState = listGatewayMetaState.list(nodesIds, null).actionGet();
    int requiredAllocation = Math.max(1, minimumMasterNodesProvider.get());
    if (nodesState.hasFailures()) {
        for (FailedNodeException failedNodeException : nodesState.failures()) {
            logger.warn("failed to fetch state from node", failedNodeException);
        }
    }
    ObjectFloatHashMap<Index> indices = new ObjectFloatHashMap<>();
    MetaData electedGlobalState = null;
    int found = 0;
    for (TransportNodesListGatewayMetaState.NodeGatewayMetaState nodeState : nodesState.getNodes()) {
        if (nodeState.metaData() == null) {
            continue;
        }
        found++;
        if (electedGlobalState == null) {
            electedGlobalState = nodeState.metaData();
        } else if (nodeState.metaData().version() > electedGlobalState.version()) {
            electedGlobalState = nodeState.metaData();
        }
        for (ObjectCursor<IndexMetaData> cursor : nodeState.metaData().indices().values()) {
            indices.addTo(cursor.value.getIndex(), 1);
        }
    }
    if (found < requiredAllocation) {
        listener.onFailure("found [" + found + "] metadata states, required [" + requiredAllocation + "]");
        return;
    }
    // update the global state, and clean the indices, we elect them in the next phase
    MetaData.Builder metaDataBuilder = MetaData.builder(electedGlobalState).removeAllIndices();
    assert !indices.containsKey(null);
    final Object[] keys = indices.keys;
    for (int i = 0; i < keys.length; i++) {
        if (keys[i] != null) {
            Index index = (Index) keys[i];
            IndexMetaData electedIndexMetaData = null;
            int indexMetaDataCount = 0;
            for (TransportNodesListGatewayMetaState.NodeGatewayMetaState nodeState : nodesState.getNodes()) {
                if (nodeState.metaData() == null) {
                    continue;
                }
                IndexMetaData indexMetaData = nodeState.metaData().index(index);
                if (indexMetaData == null) {
                    continue;
                }
                if (electedIndexMetaData == null) {
                    electedIndexMetaData = indexMetaData;
                } else if (indexMetaData.getVersion() > electedIndexMetaData.getVersion()) {
                    electedIndexMetaData = indexMetaData;
                }
                indexMetaDataCount++;
            }
            if (electedIndexMetaData != null) {
                if (indexMetaDataCount < requiredAllocation) {
                    logger.debug("[{}] found [{}], required [{}], not adding", index, indexMetaDataCount, requiredAllocation);
                }
                // TODO if this logging statement is correct then we are missing an else here
                try {
                    if (electedIndexMetaData.getState() == IndexMetaData.State.OPEN) {
                        // verify that we can actually create this index - if not we recover it as closed with lots of warn logs
                        indicesService.verifyIndexMetadata(electedIndexMetaData, electedIndexMetaData);
                    }
                } catch (Exception e) {
                    final Index electedIndex = electedIndexMetaData.getIndex();
                    logger.warn((org.apache.logging.log4j.util.Supplier<?>) () -> new ParameterizedMessage("recovering index {} failed - recovering as closed", electedIndex), e);
                    electedIndexMetaData = IndexMetaData.builder(electedIndexMetaData).state(IndexMetaData.State.CLOSE).build();
                }
                metaDataBuilder.put(electedIndexMetaData, false);
            }
        }
    }
    final ClusterSettings clusterSettings = clusterService.getClusterSettings();
    metaDataBuilder.persistentSettings(clusterSettings.archiveUnknownOrInvalidSettings(metaDataBuilder.persistentSettings(), e -> logUnknownSetting("persistent", e), (e, ex) -> logInvalidSetting("persistent", e, ex)));
    metaDataBuilder.transientSettings(clusterSettings.archiveUnknownOrInvalidSettings(metaDataBuilder.transientSettings(), e -> logUnknownSetting("transient", e), (e, ex) -> logInvalidSetting("transient", e, ex)));
    ClusterState.Builder builder = ClusterState.builder(clusterService.getClusterName());
    builder.metaData(metaDataBuilder);
    listener.onSuccess(builder.build());
}

Example 47 with IndexMetaData

use of org.elasticsearch.cluster.metadata.IndexMetaData in project elasticsearch by elastic.

the class GatewayMetaState method applyClusterState.

@Override
public void applyClusterState(ClusterChangedEvent event) {
    final ClusterState state = event.state();
    if (state.blocks().disableStatePersistence()) {
        // reset the current metadata, we need to start fresh...
        this.previousMetaData = null;
        previouslyWrittenIndices = emptySet();
        return;
    }
    MetaData newMetaData = state.metaData();
    // we don't check if metaData changed, since we might be called several times and we need to check dangling...
    Set<Index> relevantIndices = Collections.emptySet();
    boolean success = true;
    // write the state if this node is a master eligible node or if it is a data node and has shards allocated on it
    if (state.nodes().getLocalNode().isMasterNode() || state.nodes().getLocalNode().isDataNode()) {
        if (previousMetaData == null) {
            try {
                // we therefore have to check here if we have shards on disk and add their indices to the previouslyWrittenIndices list
                if (isDataOnlyNode(state)) {
                    Set<Index> newPreviouslyWrittenIndices = new HashSet<>(previouslyWrittenIndices.size());
                    for (IndexMetaData indexMetaData : newMetaData) {
                        IndexMetaData indexMetaDataOnDisk = null;
                        if (indexMetaData.getState().equals(IndexMetaData.State.CLOSE)) {
                            indexMetaDataOnDisk = metaStateService.loadIndexState(indexMetaData.getIndex());
                        }
                        if (indexMetaDataOnDisk != null) {
                            newPreviouslyWrittenIndices.add(indexMetaDataOnDisk.getIndex());
                        }
                    }
                    newPreviouslyWrittenIndices.addAll(previouslyWrittenIndices);
                    previouslyWrittenIndices = unmodifiableSet(newPreviouslyWrittenIndices);
                }
            } catch (Exception e) {
                success = false;
            }
        }
        // check if the global state changed?
        if (previousMetaData == null || !MetaData.isGlobalStateEquals(previousMetaData, newMetaData)) {
            try {
                metaStateService.writeGlobalState("changed", newMetaData);
            } catch (Exception e) {
                success = false;
            }
        }
        relevantIndices = getRelevantIndices(event.state(), event.previousState(), previouslyWrittenIndices);
        final Iterable<IndexMetaWriteInfo> writeInfo = resolveStatesToBeWritten(previouslyWrittenIndices, relevantIndices, previousMetaData, event.state().metaData());
        // check and write changes in indices
        for (IndexMetaWriteInfo indexMetaWrite : writeInfo) {
            try {
                metaStateService.writeIndex(indexMetaWrite.reason, indexMetaWrite.newMetaData);
            } catch (Exception e) {
                success = false;
            }
        }
    }
    if (success) {
        previousMetaData = newMetaData;
        previouslyWrittenIndices = unmodifiableSet(relevantIndices);
    }
}

Example 48 with IndexMetaData

use of org.elasticsearch.cluster.metadata.IndexMetaData in project elasticsearch by elastic.

the class GatewayMetaState method getRelevantIndicesOnDataOnlyNode.

public static Set<Index> getRelevantIndicesOnDataOnlyNode(ClusterState state, ClusterState previousState, Set<Index> previouslyWrittenIndices) {
    RoutingNode newRoutingNode = state.getRoutingNodes().node(state.nodes().getLocalNodeId());
    if (newRoutingNode == null) {
        throw new IllegalStateException("cluster state does not contain this node - cannot write index meta state");
    }
    Set<Index> indices = new HashSet<>();
    for (ShardRouting routing : newRoutingNode) {
        indices.add(routing.index());
    }
    // we have to check the meta data also: closed indices will not appear in the routing table, but we must still write the state if we have it written on disk previously
    for (IndexMetaData indexMetaData : state.metaData()) {
        boolean isOrWasClosed = indexMetaData.getState().equals(IndexMetaData.State.CLOSE);
        // if the index is open we might still have to write the state if it just transitioned from closed to open
        // so we have to check for that as well.
        IndexMetaData previousMetaData = previousState.metaData().index(indexMetaData.getIndex());
        if (previousMetaData != null) {
            isOrWasClosed = isOrWasClosed || previousMetaData.getState().equals(IndexMetaData.State.CLOSE);
        }
        if (previouslyWrittenIndices.contains(indexMetaData.getIndex()) && isOrWasClosed) {
            indices.add(indexMetaData.getIndex());
        }
    }
    return indices;
}

Example 49 with IndexMetaData

use of org.elasticsearch.cluster.metadata.IndexMetaData in project elasticsearch by elastic.

the class GatewayMetaState method resolveStatesToBeWritten.

/**
     * Loads the current meta state for each index in the new cluster state and checks if it has to be persisted.
     * Each index state that should be written to disk will be returned. This is only run for data only nodes.
     * It will return only the states for indices that actually have a shard allocated on the current node.
     *
     * @param previouslyWrittenIndices    A list of indices for which the state was already written before
     * @param potentiallyUnwrittenIndices The list of indices for which state should potentially be written
     * @param previousMetaData            The last meta data we know of. meta data for all indices in previouslyWrittenIndices list is persisted now
     * @param newMetaData                 The new metadata
     * @return iterable over all indices states that should be written to disk
     */
public static Iterable<GatewayMetaState.IndexMetaWriteInfo> resolveStatesToBeWritten(Set<Index> previouslyWrittenIndices, Set<Index> potentiallyUnwrittenIndices, MetaData previousMetaData, MetaData newMetaData) {
    List<GatewayMetaState.IndexMetaWriteInfo> indicesToWrite = new ArrayList<>();
    for (Index index : potentiallyUnwrittenIndices) {
        IndexMetaData newIndexMetaData = newMetaData.getIndexSafe(index);
        IndexMetaData previousIndexMetaData = previousMetaData == null ? null : previousMetaData.index(index);
        String writeReason = null;
        if (previouslyWrittenIndices.contains(index) == false || previousIndexMetaData == null) {
            writeReason = "freshly created";
        } else if (previousIndexMetaData.getVersion() != newIndexMetaData.getVersion()) {
            writeReason = "version changed from [" + previousIndexMetaData.getVersion() + "] to [" + newIndexMetaData.getVersion() + "]";
        }
        if (writeReason != null) {
            indicesToWrite.add(new GatewayMetaState.IndexMetaWriteInfo(newIndexMetaData, previousIndexMetaData, writeReason));
        }
    }
    return indicesToWrite;
}

Example 50 with IndexMetaData

use of org.elasticsearch.cluster.metadata.IndexMetaData in project elasticsearch by elastic.

the class GatewayMetaState method getRelevantIndicesForMasterEligibleNode.

public static Set<Index> getRelevantIndicesForMasterEligibleNode(ClusterState state) {
    Set<Index> relevantIndices;
    relevantIndices = new HashSet<>();
    // we have to iterate over the metadata to make sure we also capture closed indices
    for (IndexMetaData indexMetaData : state.metaData()) {
        relevantIndices.add(indexMetaData.getIndex());
    }
    return relevantIndices;
}