Examples with Metadata org.elasticsearch.cluster.metadata.Metadata used on opensource projects

Example 36 with Metadata

use of org.elasticsearch.cluster.metadata.Metadata in project elasticsearch by elastic.

the class SnapshotsService method beginSnapshot.

/**
     * Starts snapshot.
     * <p>
     * Creates snapshot in repository and updates snapshot metadata record with list of shards that needs to be processed.
     *
     * @param clusterState               cluster state
     * @param snapshot                   snapshot meta data
     * @param partial                    allow partial snapshots
     * @param userCreateSnapshotListener listener
     */
private void beginSnapshot(final ClusterState clusterState, final SnapshotsInProgress.Entry snapshot, final boolean partial, final CreateSnapshotListener userCreateSnapshotListener) {
    boolean snapshotCreated = false;
    try {
        Repository repository = repositoriesService.repository(snapshot.snapshot().getRepository());
        MetaData metaData = clusterState.metaData();
        if (!snapshot.includeGlobalState()) {
            // Remove global state from the cluster state
            MetaData.Builder builder = MetaData.builder();
            for (IndexId index : snapshot.indices()) {
                builder.put(metaData.index(index.getName()), false);
            }
            metaData = builder.build();
        }
        repository.initializeSnapshot(snapshot.snapshot().getSnapshotId(), snapshot.indices(), metaData);
        snapshotCreated = true;
        if (snapshot.indices().isEmpty()) {
            // No indices in this snapshot - we are done
            userCreateSnapshotListener.onResponse();
            endSnapshot(snapshot);
            return;
        }
        clusterService.submitStateUpdateTask("update_snapshot [" + snapshot.snapshot() + "]", new ClusterStateUpdateTask() {

            boolean accepted = false;

            SnapshotsInProgress.Entry updatedSnapshot;

            String failure = null;

            @Override
            public ClusterState execute(ClusterState currentState) {
                SnapshotsInProgress snapshots = currentState.custom(SnapshotsInProgress.TYPE);
                List<SnapshotsInProgress.Entry> entries = new ArrayList<>();
                for (SnapshotsInProgress.Entry entry : snapshots.entries()) {
                    if (entry.snapshot().equals(snapshot.snapshot())) {
                        // Replace the snapshot that was just created
                        ImmutableOpenMap<ShardId, SnapshotsInProgress.ShardSnapshotStatus> shards = shards(currentState, entry.indices());
                        if (!partial) {
                            Tuple<Set<String>, Set<String>> indicesWithMissingShards = indicesWithMissingShards(shards, currentState.metaData());
                            Set<String> missing = indicesWithMissingShards.v1();
                            Set<String> closed = indicesWithMissingShards.v2();
                            if (missing.isEmpty() == false || closed.isEmpty() == false) {
                                StringBuilder failureMessage = new StringBuilder();
                                updatedSnapshot = new SnapshotsInProgress.Entry(entry, State.FAILED, shards);
                                entries.add(updatedSnapshot);
                                if (missing.isEmpty() == false) {
                                    failureMessage.append("Indices don't have primary shards ");
                                    failureMessage.append(missing);
                                }
                                if (closed.isEmpty() == false) {
                                    if (failureMessage.length() > 0) {
                                        failureMessage.append("; ");
                                    }
                                    failureMessage.append("Indices are closed ");
                                    failureMessage.append(closed);
                                }
                                failure = failureMessage.toString();
                                continue;
                            }
                        }
                        updatedSnapshot = new SnapshotsInProgress.Entry(entry, State.STARTED, shards);
                        entries.add(updatedSnapshot);
                        if (!completed(shards.values())) {
                            accepted = true;
                        }
                    } else {
                        entries.add(entry);
                    }
                }
                return ClusterState.builder(currentState).putCustom(SnapshotsInProgress.TYPE, new SnapshotsInProgress(Collections.unmodifiableList(entries))).build();
            }

            @Override
            public void onFailure(String source, Exception e) {
                logger.warn((Supplier<?>) () -> new ParameterizedMessage("[{}] failed to create snapshot", snapshot.snapshot().getSnapshotId()), e);
                removeSnapshotFromClusterState(snapshot.snapshot(), null, e, new CleanupAfterErrorListener(snapshot, true, userCreateSnapshotListener, e));
            }

            @Override
            public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
                // The userCreateSnapshotListener.onResponse() notifies caller that the snapshot was accepted
                // for processing. If client wants to wait for the snapshot completion, it can register snapshot
                // completion listener in this method. For the snapshot completion to work properly, the snapshot
                // should still exist when listener is registered.
                userCreateSnapshotListener.onResponse();
                // go ahead and continue working on this snapshot rather then end here.
                if (!accepted && updatedSnapshot != null) {
                    endSnapshot(updatedSnapshot, failure);
                }
            }
        });
    } catch (Exception e) {
        logger.warn((Supplier<?>) () -> new ParameterizedMessage("failed to create snapshot [{}]", snapshot.snapshot().getSnapshotId()), e);
        removeSnapshotFromClusterState(snapshot.snapshot(), null, e, new CleanupAfterErrorListener(snapshot, snapshotCreated, userCreateSnapshotListener, e));
    }
}

Example 37 with Metadata

use of org.elasticsearch.cluster.metadata.Metadata in project elasticsearch by elastic.

the class ClusterStateHealthTests method generateClusterStates.

private List<ClusterState> generateClusterStates(final ClusterState originalClusterState, final String indexName, final int numberOfReplicas, final boolean withPrimaryAllocationFailures) {
    // generate random node ids
    final Set<String> nodeIds = new HashSet<>();
    final int numNodes = randomIntBetween(numberOfReplicas + 1, 10);
    for (int i = 0; i < numNodes; i++) {
        nodeIds.add(randomAsciiOfLength(8));
    }
    final List<ClusterState> clusterStates = new ArrayList<>();
    clusterStates.add(originalClusterState);
    ClusterState clusterState = originalClusterState;
    // initialize primaries
    RoutingTable routingTable = originalClusterState.routingTable();
    IndexRoutingTable indexRoutingTable = routingTable.index(indexName);
    IndexRoutingTable.Builder newIndexRoutingTable = IndexRoutingTable.builder(indexRoutingTable.getIndex());
    for (final ObjectCursor<IndexShardRoutingTable> shardEntry : indexRoutingTable.getShards().values()) {
        final IndexShardRoutingTable shardRoutingTable = shardEntry.value;
        for (final ShardRouting shardRouting : shardRoutingTable.getShards()) {
            if (shardRouting.primary()) {
                newIndexRoutingTable.addShard(shardRouting.initialize(randomFrom(nodeIds), null, shardRouting.getExpectedShardSize()));
            } else {
                newIndexRoutingTable.addShard(shardRouting);
            }
        }
    }
    routingTable = RoutingTable.builder(routingTable).add(newIndexRoutingTable).build();
    clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
    clusterStates.add(clusterState);
    // some primaries started
    indexRoutingTable = routingTable.index(indexName);
    newIndexRoutingTable = IndexRoutingTable.builder(indexRoutingTable.getIndex());
    ImmutableOpenIntMap.Builder<Set<String>> allocationIds = ImmutableOpenIntMap.<Set<String>>builder();
    for (final ObjectCursor<IndexShardRoutingTable> shardEntry : indexRoutingTable.getShards().values()) {
        final IndexShardRoutingTable shardRoutingTable = shardEntry.value;
        for (final ShardRouting shardRouting : shardRoutingTable.getShards()) {
            if (shardRouting.primary() && randomBoolean()) {
                final ShardRouting newShardRouting = shardRouting.moveToStarted();
                allocationIds.fPut(newShardRouting.getId(), Sets.newHashSet(newShardRouting.allocationId().getId()));
                newIndexRoutingTable.addShard(newShardRouting);
            } else {
                newIndexRoutingTable.addShard(shardRouting);
            }
        }
    }
    routingTable = RoutingTable.builder(routingTable).add(newIndexRoutingTable).build();
    IndexMetaData.Builder idxMetaBuilder = IndexMetaData.builder(clusterState.metaData().index(indexName));
    for (final IntObjectCursor<Set<String>> entry : allocationIds.build()) {
        idxMetaBuilder.putInSyncAllocationIds(entry.key, entry.value);
    }
    MetaData.Builder metaDataBuilder = MetaData.builder(clusterState.metaData()).put(idxMetaBuilder);
    clusterState = ClusterState.builder(clusterState).routingTable(routingTable).metaData(metaDataBuilder).build();
    clusterStates.add(clusterState);
    if (withPrimaryAllocationFailures) {
        boolean alreadyFailedPrimary = false;
        // some primaries failed to allocate
        indexRoutingTable = routingTable.index(indexName);
        newIndexRoutingTable = IndexRoutingTable.builder(indexRoutingTable.getIndex());
        for (final ObjectCursor<IndexShardRoutingTable> shardEntry : indexRoutingTable.getShards().values()) {
            final IndexShardRoutingTable shardRoutingTable = shardEntry.value;
            for (final ShardRouting shardRouting : shardRoutingTable.getShards()) {
                if (shardRouting.primary() && (shardRouting.started() == false || alreadyFailedPrimary == false)) {
                    newIndexRoutingTable.addShard(shardRouting.moveToUnassigned(new UnassignedInfo(UnassignedInfo.Reason.ALLOCATION_FAILED, "unlucky shard")));
                    alreadyFailedPrimary = true;
                } else {
                    newIndexRoutingTable.addShard(shardRouting);
                }
            }
        }
        routingTable = RoutingTable.builder(routingTable).add(newIndexRoutingTable).build();
        clusterStates.add(ClusterState.builder(clusterState).routingTable(routingTable).build());
        return clusterStates;
    }
    // all primaries started
    indexRoutingTable = routingTable.index(indexName);
    newIndexRoutingTable = IndexRoutingTable.builder(indexRoutingTable.getIndex());
    allocationIds = ImmutableOpenIntMap.<Set<String>>builder();
    for (final ObjectCursor<IndexShardRoutingTable> shardEntry : indexRoutingTable.getShards().values()) {
        final IndexShardRoutingTable shardRoutingTable = shardEntry.value;
        for (final ShardRouting shardRouting : shardRoutingTable.getShards()) {
            if (shardRouting.primary() && shardRouting.started() == false) {
                final ShardRouting newShardRouting = shardRouting.moveToStarted();
                allocationIds.fPut(newShardRouting.getId(), Sets.newHashSet(newShardRouting.allocationId().getId()));
                newIndexRoutingTable.addShard(newShardRouting);
            } else {
                newIndexRoutingTable.addShard(shardRouting);
            }
        }
    }
    routingTable = RoutingTable.builder(routingTable).add(newIndexRoutingTable).build();
    idxMetaBuilder = IndexMetaData.builder(clusterState.metaData().index(indexName));
    for (final IntObjectCursor<Set<String>> entry : allocationIds.build()) {
        idxMetaBuilder.putInSyncAllocationIds(entry.key, entry.value);
    }
    metaDataBuilder = MetaData.builder(clusterState.metaData()).put(idxMetaBuilder);
    clusterState = ClusterState.builder(clusterState).routingTable(routingTable).metaData(metaDataBuilder).build();
    clusterStates.add(clusterState);
    // initialize replicas
    indexRoutingTable = routingTable.index(indexName);
    newIndexRoutingTable = IndexRoutingTable.builder(indexRoutingTable.getIndex());
    for (final ObjectCursor<IndexShardRoutingTable> shardEntry : indexRoutingTable.getShards().values()) {
        final IndexShardRoutingTable shardRoutingTable = shardEntry.value;
        final String primaryNodeId = shardRoutingTable.primaryShard().currentNodeId();
        Set<String> allocatedNodes = new HashSet<>();
        allocatedNodes.add(primaryNodeId);
        for (final ShardRouting shardRouting : shardRoutingTable.getShards()) {
            if (shardRouting.primary() == false) {
                // give the replica a different node id than the primary
                String replicaNodeId = randomFrom(Sets.difference(nodeIds, allocatedNodes));
                newIndexRoutingTable.addShard(shardRouting.initialize(replicaNodeId, null, shardRouting.getExpectedShardSize()));
                allocatedNodes.add(replicaNodeId);
            } else {
                newIndexRoutingTable.addShard(shardRouting);
            }
        }
    }
    routingTable = RoutingTable.builder(routingTable).add(newIndexRoutingTable).build();
    clusterStates.add(ClusterState.builder(clusterState).routingTable(routingTable).build());
    // some replicas started
    indexRoutingTable = routingTable.index(indexName);
    newIndexRoutingTable = IndexRoutingTable.builder(indexRoutingTable.getIndex());
    for (final ObjectCursor<IndexShardRoutingTable> shardEntry : indexRoutingTable.getShards().values()) {
        final IndexShardRoutingTable shardRoutingTable = shardEntry.value;
        for (final ShardRouting shardRouting : shardRoutingTable.getShards()) {
            if (shardRouting.primary() == false && randomBoolean()) {
                newIndexRoutingTable.addShard(shardRouting.moveToStarted());
            } else {
                newIndexRoutingTable.addShard(shardRouting);
            }
        }
    }
    routingTable = RoutingTable.builder(routingTable).add(newIndexRoutingTable).build();
    clusterStates.add(ClusterState.builder(clusterState).routingTable(routingTable).build());
    // all replicas started
    boolean replicaStateChanged = false;
    indexRoutingTable = routingTable.index(indexName);
    newIndexRoutingTable = IndexRoutingTable.builder(indexRoutingTable.getIndex());
    for (final ObjectCursor<IndexShardRoutingTable> shardEntry : indexRoutingTable.getShards().values()) {
        final IndexShardRoutingTable shardRoutingTable = shardEntry.value;
        for (final ShardRouting shardRouting : shardRoutingTable.getShards()) {
            if (shardRouting.primary() == false && shardRouting.started() == false) {
                newIndexRoutingTable.addShard(shardRouting.moveToStarted());
                replicaStateChanged = true;
            } else {
                newIndexRoutingTable.addShard(shardRouting);
            }
        }
    }
    // all of the replicas may have moved to started in the previous phase already
    if (replicaStateChanged) {
        routingTable = RoutingTable.builder(routingTable).add(newIndexRoutingTable).build();
        clusterStates.add(ClusterState.builder(clusterState).routingTable(routingTable).build());
    }
    return clusterStates;
}

Example 38 with Metadata

use of org.elasticsearch.cluster.metadata.Metadata in project elasticsearch by elastic.

the class ClusterStateHealthTests method testClusterHealth.

public void testClusterHealth() throws IOException {
    RoutingTableGenerator routingTableGenerator = new RoutingTableGenerator();
    RoutingTableGenerator.ShardCounter counter = new RoutingTableGenerator.ShardCounter();
    RoutingTable.Builder routingTable = RoutingTable.builder();
    MetaData.Builder metaData = MetaData.builder();
    for (int i = randomInt(4); i >= 0; i--) {
        int numberOfShards = randomInt(3) + 1;
        int numberOfReplicas = randomInt(4);
        IndexMetaData indexMetaData = IndexMetaData.builder("test_" + Integer.toString(i)).settings(settings(Version.CURRENT)).numberOfShards(numberOfShards).numberOfReplicas(numberOfReplicas).build();
        IndexRoutingTable indexRoutingTable = routingTableGenerator.genIndexRoutingTable(indexMetaData, counter);
        metaData.put(indexMetaData, true);
        routingTable.add(indexRoutingTable);
    }
    ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable.build()).build();
    String[] concreteIndices = indexNameExpressionResolver.concreteIndexNames(clusterState, IndicesOptions.strictExpand(), (String[]) null);
    ClusterStateHealth clusterStateHealth = new ClusterStateHealth(clusterState, concreteIndices);
    logger.info("cluster status: {}, expected {}", clusterStateHealth.getStatus(), counter.status());
    clusterStateHealth = maybeSerialize(clusterStateHealth);
    assertClusterHealth(clusterStateHealth, counter);
}

Example 39 with Metadata

use of org.elasticsearch.cluster.metadata.Metadata in project elasticsearch by elastic.

the class AllocationBenchmark method setUp.

@Setup
public void setUp() throws Exception {
    final String[] params = indicesShardsReplicasNodes.split("\\|");
    int numIndices = toInt(params[0]);
    int numShards = toInt(params[1]);
    int numReplicas = toInt(params[2]);
    int numNodes = toInt(params[3]);
    strategy = Allocators.createAllocationService(Settings.builder().put("cluster.routing.allocation.awareness.attributes", "tag").build());
    MetaData.Builder mb = MetaData.builder();
    for (int i = 1; i <= numIndices; i++) {
        mb.put(IndexMetaData.builder("test_" + i).settings(Settings.builder().put("index.version.created", Version.CURRENT)).numberOfShards(numShards).numberOfReplicas(numReplicas));
    }
    MetaData metaData = mb.build();
    RoutingTable.Builder rb = RoutingTable.builder();
    for (int i = 1; i <= numIndices; i++) {
        rb.addAsNew(metaData.index("test_" + i));
    }
    RoutingTable routingTable = rb.build();
    DiscoveryNodes.Builder nb = DiscoveryNodes.builder();
    for (int i = 1; i <= numNodes; i++) {
        nb.add(Allocators.newNode("node" + i, Collections.singletonMap("tag", "tag_" + (i % numTags))));
    }
    initialClusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).nodes(nb).build();
}

Example 40 with Metadata

use of org.elasticsearch.cluster.metadata.Metadata in project elasticsearch by elastic.

the class TransportRolloverAction method masterOperation.

@Override
protected void masterOperation(final RolloverRequest rolloverRequest, final ClusterState state, final ActionListener<RolloverResponse> listener) {
    final MetaData metaData = state.metaData();
    validate(metaData, rolloverRequest);
    final AliasOrIndex aliasOrIndex = metaData.getAliasAndIndexLookup().get(rolloverRequest.getAlias());
    final IndexMetaData indexMetaData = aliasOrIndex.getIndices().get(0);
    final String sourceProvidedName = indexMetaData.getSettings().get(IndexMetaData.SETTING_INDEX_PROVIDED_NAME, indexMetaData.getIndex().getName());
    final String sourceIndexName = indexMetaData.getIndex().getName();
    final String unresolvedName = (rolloverRequest.getNewIndexName() != null) ? rolloverRequest.getNewIndexName() : generateRolloverIndexName(sourceProvidedName, indexNameExpressionResolver);
    final String rolloverIndexName = indexNameExpressionResolver.resolveDateMathExpression(unresolvedName);
    // will fail if the index already exists
    MetaDataCreateIndexService.validateIndexName(rolloverIndexName, state);
    client.admin().indices().prepareStats(sourceIndexName).clear().setDocs(true).execute(new ActionListener<IndicesStatsResponse>() {

        @Override
        public void onResponse(IndicesStatsResponse statsResponse) {
            final Set<Condition.Result> conditionResults = evaluateConditions(rolloverRequest.getConditions(), statsResponse.getTotal().getDocs(), metaData.index(sourceIndexName));
            if (rolloverRequest.isDryRun()) {
                listener.onResponse(new RolloverResponse(sourceIndexName, rolloverIndexName, conditionResults, true, false, false, false));
                return;
            }
            if (conditionResults.size() == 0 || conditionResults.stream().anyMatch(result -> result.matched)) {
                CreateIndexClusterStateUpdateRequest updateRequest = prepareCreateIndexRequest(unresolvedName, rolloverIndexName, rolloverRequest);
                createIndexService.createIndex(updateRequest, ActionListener.wrap(createIndexClusterStateUpdateResponse -> {
                    indexAliasesService.indicesAliases(prepareRolloverAliasesUpdateRequest(sourceIndexName, rolloverIndexName, rolloverRequest), ActionListener.wrap(aliasClusterStateUpdateResponse -> {
                        if (aliasClusterStateUpdateResponse.isAcknowledged()) {
                            activeShardsObserver.waitForActiveShards(rolloverIndexName, rolloverRequest.getCreateIndexRequest().waitForActiveShards(), rolloverRequest.masterNodeTimeout(), isShardsAcked -> listener.onResponse(new RolloverResponse(sourceIndexName, rolloverIndexName, conditionResults, false, true, true, isShardsAcked)), listener::onFailure);
                        } else {
                            listener.onResponse(new RolloverResponse(sourceIndexName, rolloverIndexName, conditionResults, false, true, false, false));
                        }
                    }, listener::onFailure));
                }, listener::onFailure));
            } else {
                // conditions not met
                listener.onResponse(new RolloverResponse(sourceIndexName, rolloverIndexName, conditionResults, false, false, false, false));
            }
        }

        @Override
        public void onFailure(Exception e) {
            listener.onFailure(e);
        }
    });
}