Examples with IndexMetadata org.opensearch.cluster.metadata.IndexMetadata used on opensource projects

Example 21 with IndexMetadata

use of org.opensearch.cluster.metadata.IndexMetadata in project OpenSearch by opensearch-project.

the class IndexShard method startRecovery.

public void startRecovery(RecoveryState recoveryState, PeerRecoveryTargetService recoveryTargetService, PeerRecoveryTargetService.RecoveryListener recoveryListener, RepositoriesService repositoriesService, Consumer<MappingMetadata> mappingUpdateConsumer, IndicesService indicesService) {
    // }
    assert recoveryState.getRecoverySource().equals(shardRouting.recoverySource());
    switch(recoveryState.getRecoverySource().getType()) {
        case EMPTY_STORE:
        case EXISTING_STORE:
            executeRecovery("from store", recoveryState, recoveryListener, this::recoverFromStore);
            break;
        case PEER:
            try {
                markAsRecovering("from " + recoveryState.getSourceNode(), recoveryState);
                recoveryTargetService.startRecovery(this, recoveryState.getSourceNode(), recoveryListener);
            } catch (Exception e) {
                failShard("corrupted preexisting index", e);
                recoveryListener.onRecoveryFailure(recoveryState, new RecoveryFailedException(recoveryState, null, e), true);
            }
            break;
        case SNAPSHOT:
            final String repo = ((SnapshotRecoverySource) recoveryState.getRecoverySource()).snapshot().getRepository();
            executeRecovery("from snapshot", recoveryState, recoveryListener, l -> restoreFromRepository(repositoriesService.repository(repo), l));
            break;
        case LOCAL_SHARDS:
            final IndexMetadata indexMetadata = indexSettings().getIndexMetadata();
            final Index resizeSourceIndex = indexMetadata.getResizeSourceIndex();
            final List<IndexShard> startedShards = new ArrayList<>();
            final IndexService sourceIndexService = indicesService.indexService(resizeSourceIndex);
            final Set<ShardId> requiredShards;
            final int numShards;
            if (sourceIndexService != null) {
                requiredShards = IndexMetadata.selectRecoverFromShards(shardId().id(), sourceIndexService.getMetadata(), indexMetadata.getNumberOfShards());
                for (IndexShard shard : sourceIndexService) {
                    if (shard.state() == IndexShardState.STARTED && requiredShards.contains(shard.shardId())) {
                        startedShards.add(shard);
                    }
                }
                numShards = requiredShards.size();
            } else {
                numShards = -1;
                requiredShards = Collections.emptySet();
            }
            if (numShards == startedShards.size()) {
                assert requiredShards.isEmpty() == false;
                executeRecovery("from local shards", recoveryState, recoveryListener, l -> recoverFromLocalShards(mappingUpdateConsumer, startedShards.stream().filter((s) -> requiredShards.contains(s.shardId())).collect(Collectors.toList()), l));
            } else {
                final RuntimeException e;
                if (numShards == -1) {
                    e = new IndexNotFoundException(resizeSourceIndex);
                } else {
                    e = new IllegalStateException("not all required shards of index " + resizeSourceIndex + " are started yet, expected " + numShards + " found " + startedShards.size() + " can't recover shard " + shardId());
                }
                throw e;
            }
            break;
        default:
            throw new IllegalArgumentException("Unknown recovery source " + recoveryState.getRecoverySource());
    }
}

Example 22 with IndexMetadata

use of org.opensearch.cluster.metadata.IndexMetadata in project OpenSearch by opensearch-project.

the class TransportShardBulkAction method executeBulkItemRequest.

/**
 * Executes bulk item requests and handles request execution exceptions.
 * @return {@code true} if request completed on this thread and the listener was invoked, {@code false} if the request triggered
 *                      a mapping update that will finish and invoke the listener on a different thread
 */
static boolean executeBulkItemRequest(BulkPrimaryExecutionContext context, UpdateHelper updateHelper, LongSupplier nowInMillisSupplier, MappingUpdatePerformer mappingUpdater, Consumer<ActionListener<Void>> waitForMappingUpdate, ActionListener<Void> itemDoneListener) throws Exception {
    final DocWriteRequest.OpType opType = context.getCurrent().opType();
    final UpdateHelper.Result updateResult;
    if (opType == DocWriteRequest.OpType.UPDATE) {
        final UpdateRequest updateRequest = (UpdateRequest) context.getCurrent();
        try {
            updateResult = updateHelper.prepare(updateRequest, context.getPrimary(), nowInMillisSupplier);
        } catch (Exception failure) {
            // we may fail translating a update to index or delete operation
            // we use index result to communicate failure while translating update request
            final Engine.Result result = new Engine.IndexResult(failure, updateRequest.version());
            context.setRequestToExecute(updateRequest);
            context.markOperationAsExecuted(result);
            context.markAsCompleted(context.getExecutionResult());
            return true;
        }
        // execute translated update request
        switch(updateResult.getResponseResult()) {
            case CREATED:
            case UPDATED:
                IndexRequest indexRequest = updateResult.action();
                IndexMetadata metadata = context.getPrimary().indexSettings().getIndexMetadata();
                MappingMetadata mappingMd = metadata.mapping();
                indexRequest.process(metadata.getCreationVersion(), mappingMd, updateRequest.concreteIndex());
                context.setRequestToExecute(indexRequest);
                break;
            case DELETED:
                context.setRequestToExecute(updateResult.action());
                break;
            case NOOP:
                context.markOperationAsNoOp(updateResult.action());
                context.markAsCompleted(context.getExecutionResult());
                return true;
            default:
                throw new IllegalStateException("Illegal update operation " + updateResult.getResponseResult());
        }
    } else {
        context.setRequestToExecute(context.getCurrent());
        updateResult = null;
    }
    // also checks that we're in TRANSLATED state
    assert context.getRequestToExecute() != null;
    final IndexShard primary = context.getPrimary();
    final long version = context.getRequestToExecute().version();
    final boolean isDelete = context.getRequestToExecute().opType() == DocWriteRequest.OpType.DELETE;
    final Engine.Result result;
    if (isDelete) {
        final DeleteRequest request = context.getRequestToExecute();
        result = primary.applyDeleteOperationOnPrimary(version, request.id(), request.versionType(), request.ifSeqNo(), request.ifPrimaryTerm());
    } else {
        final IndexRequest request = context.getRequestToExecute();
        result = primary.applyIndexOperationOnPrimary(version, request.versionType(), new SourceToParse(request.index(), request.id(), request.source(), request.getContentType(), request.routing()), request.ifSeqNo(), request.ifPrimaryTerm(), request.getAutoGeneratedTimestamp(), request.isRetry());
    }
    if (result.getResultType() == Engine.Result.Type.MAPPING_UPDATE_REQUIRED) {
        try {
            primary.mapperService().merge(MapperService.SINGLE_MAPPING_NAME, new CompressedXContent(result.getRequiredMappingUpdate(), ToXContent.EMPTY_PARAMS), MapperService.MergeReason.MAPPING_UPDATE_PREFLIGHT);
        } catch (Exception e) {
            logger.info(() -> new ParameterizedMessage("{} mapping update rejected by primary", primary.shardId()), e);
            onComplete(exceptionToResult(e, primary, isDelete, version), context, updateResult);
            return true;
        }
        mappingUpdater.updateMappings(result.getRequiredMappingUpdate(), primary.shardId(), new ActionListener<Void>() {

            @Override
            public void onResponse(Void v) {
                context.markAsRequiringMappingUpdate();
                waitForMappingUpdate.accept(ActionListener.runAfter(new ActionListener<Void>() {

                    @Override
                    public void onResponse(Void v) {
                        assert context.requiresWaitingForMappingUpdate();
                        context.resetForExecutionForRetry();
                    }

                    @Override
                    public void onFailure(Exception e) {
                        context.failOnMappingUpdate(e);
                    }
                }, () -> itemDoneListener.onResponse(null)));
            }

            @Override
            public void onFailure(Exception e) {
                onComplete(exceptionToResult(e, primary, isDelete, version), context, updateResult);
                // Requesting mapping update failed, so we don't have to wait for a cluster state update
                assert context.isInitial();
                itemDoneListener.onResponse(null);
            }
        });
        return false;
    } else {
        onComplete(result, context, updateResult);
    }
    return true;
}

Example 23 with IndexMetadata

use of org.opensearch.cluster.metadata.IndexMetadata in project OpenSearch by opensearch-project.

the class ClusterState method toString.

@Override
public String toString() {
    StringBuilder sb = new StringBuilder();
    final String TAB = "   ";
    sb.append("cluster uuid: ").append(metadata.clusterUUID()).append(" [committed: ").append(metadata.clusterUUIDCommitted()).append("]").append("\n");
    sb.append("version: ").append(version).append("\n");
    sb.append("state uuid: ").append(stateUUID).append("\n");
    sb.append("from_diff: ").append(wasReadFromDiff).append("\n");
    sb.append("meta data version: ").append(metadata.version()).append("\n");
    sb.append(TAB).append("coordination_metadata:\n");
    sb.append(TAB).append(TAB).append("term: ").append(coordinationMetadata().term()).append("\n");
    sb.append(TAB).append(TAB).append("last_committed_config: ").append(coordinationMetadata().getLastCommittedConfiguration()).append("\n");
    sb.append(TAB).append(TAB).append("last_accepted_config: ").append(coordinationMetadata().getLastAcceptedConfiguration()).append("\n");
    sb.append(TAB).append(TAB).append("voting tombstones: ").append(coordinationMetadata().getVotingConfigExclusions()).append("\n");
    for (IndexMetadata indexMetadata : metadata) {
        sb.append(TAB).append(indexMetadata.getIndex());
        sb.append(": v[").append(indexMetadata.getVersion()).append("], mv[").append(indexMetadata.getMappingVersion()).append("], sv[").append(indexMetadata.getSettingsVersion()).append("], av[").append(indexMetadata.getAliasesVersion()).append("]\n");
        for (int shard = 0; shard < indexMetadata.getNumberOfShards(); shard++) {
            sb.append(TAB).append(TAB).append(shard).append(": ");
            sb.append("p_term [").append(indexMetadata.primaryTerm(shard)).append("], ");
            sb.append("isa_ids ").append(indexMetadata.inSyncAllocationIds(shard)).append("\n");
        }
    }
    if (metadata.customs().isEmpty() == false) {
        sb.append("metadata customs:\n");
        for (final ObjectObjectCursor<String, Metadata.Custom> cursor : metadata.customs()) {
            final String type = cursor.key;
            final Metadata.Custom custom = cursor.value;
            sb.append(TAB).append(type).append(": ").append(custom);
        }
        sb.append("\n");
    }
    sb.append(blocks());
    sb.append(nodes());
    sb.append(routingTable());
    sb.append(getRoutingNodes());
    if (customs.isEmpty() == false) {
        sb.append("customs:\n");
        for (ObjectObjectCursor<String, Custom> cursor : customs) {
            final String type = cursor.key;
            final Custom custom = cursor.value;
            sb.append(TAB).append(type).append(": ").append(custom);
        }
    }
    return sb.toString();
}

Example 24 with IndexMetadata

use of org.opensearch.cluster.metadata.IndexMetadata in project OpenSearch by opensearch-project.

the class ClusterChangedEvent method indicesDeletedFromClusterState.

// Get the deleted indices by comparing the index metadatas in the previous and new cluster states.
// If an index exists in the previous cluster state, but not in the new cluster state, it must have been deleted.
private List<Index> indicesDeletedFromClusterState() {
    // https://github.com/elastic/elasticsearch/issues/11665
    if (metadataChanged() == false || isNewCluster()) {
        return Collections.emptyList();
    }
    Set<Index> deleted = null;
    final Metadata previousMetadata = previousState.metadata();
    final Metadata currentMetadata = state.metadata();
    for (ObjectCursor<IndexMetadata> cursor : previousMetadata.indices().values()) {
        IndexMetadata index = cursor.value;
        IndexMetadata current = currentMetadata.index(index.getIndex());
        if (current == null) {
            if (deleted == null) {
                deleted = new HashSet<>();
            }
            deleted.add(index.getIndex());
        }
    }
    final IndexGraveyard currentGraveyard = currentMetadata.indexGraveyard();
    final IndexGraveyard previousGraveyard = previousMetadata.indexGraveyard();
    // each node should make sure to delete any related data.
    if (currentGraveyard != previousGraveyard) {
        final IndexGraveyardDiff indexGraveyardDiff = (IndexGraveyardDiff) currentGraveyard.diff(previousGraveyard);
        final List<IndexGraveyard.Tombstone> added = indexGraveyardDiff.getAdded();
        if (added.isEmpty() == false) {
            if (deleted == null) {
                deleted = new HashSet<>();
            }
            for (IndexGraveyard.Tombstone tombstone : added) {
                deleted.add(tombstone.getIndex());
            }
        }
    }
    return deleted == null ? Collections.<Index>emptyList() : new ArrayList<>(deleted);
}

Example 25 with IndexMetadata

use of org.opensearch.cluster.metadata.IndexMetadata in project OpenSearch by opensearch-project.

the class FilterAllocationDecider method canAllocate.

@Override
public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
    if (shardRouting.unassigned()) {
        // only for unassigned - we filter allocation right after the index creation (for shard shrinking) to ensure
        // that once it has been allocated post API the replicas can be allocated elsewhere without user interaction
        // this is a setting that can only be set within the system!
        IndexMetadata indexMd = allocation.metadata().getIndexSafe(shardRouting.index());
        DiscoveryNodeFilters initialRecoveryFilters = DiscoveryNodeFilters.trimTier(indexMd.getInitialRecoveryFilters());
        if (initialRecoveryFilters != null && shardRouting.recoverySource().getType() == RecoverySource.Type.LOCAL_SHARDS && initialRecoveryFilters.match(node.node()) == false) {
            String explanation = "initial allocation of the shrunken index is only allowed on nodes [%s] that hold a copy of every shard in the index";
            return allocation.decision(Decision.NO, NAME, explanation, initialRecoveryFilters);
        }
    }
    return shouldFilter(shardRouting, node.node(), allocation);
}