Examples with Metadata org.elasticsearch.cluster.metadata.Metadata used on opensource projects

Example 91 with Metadata

use of org.elasticsearch.cluster.metadata.Metadata in project elasticsearch by elastic.

the class SingleShardNoReplicasRoutingTests method testSingleIndexShardFailed.

public void testSingleIndexShardFailed() {
    AllocationService strategy = createAllocationService(Settings.builder().put("cluster.routing.allocation.node_concurrent_recoveries", 10).build());
    logger.info("Building initial routing table");
    MetaData metaData = MetaData.builder().put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0)).build();
    RoutingTable.Builder routingTableBuilder = RoutingTable.builder().addAsNew(metaData.index("test"));
    ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTableBuilder.build()).build();
    assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().get(0).state(), equalTo(UNASSIGNED));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().get(0).currentNodeId(), nullValue());
    logger.info("Adding one node and rerouting");
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1"))).build();
    ClusterState newState = strategy.reroute(clusterState, "reroute");
    assertThat(newState, not(equalTo(clusterState)));
    clusterState = newState;
    assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().get(0).unassigned(), equalTo(false));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().get(0).state(), equalTo(INITIALIZING));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().get(0).currentNodeId(), equalTo("node1"));
    logger.info("Marking the shard as failed");
    RoutingNodes routingNodes = clusterState.getRoutingNodes();
    newState = strategy.applyFailedShard(clusterState, routingNodes.node("node1").shardsWithState(INITIALIZING).get(0));
    assertThat(newState, not(equalTo(clusterState)));
    clusterState = newState;
    assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().get(0).state(), equalTo(UNASSIGNED));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().get(0).currentNodeId(), nullValue());
}

Example 92 with Metadata

use of org.elasticsearch.cluster.metadata.Metadata in project elasticsearch by elastic.

the class IndexBalanceTests method testBalanceAllNodesStartedAddIndex.

public void testBalanceAllNodesStartedAddIndex() {
    AllocationService strategy = createAllocationService(Settings.builder().put("cluster.routing.allocation.node_concurrent_recoveries", 10).put("cluster.routing.allocation.node_initial_primaries_recoveries", 10).put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING.getKey(), "always").put("cluster.routing.allocation.cluster_concurrent_rebalance", -1).build());
    logger.info("Building initial routing table");
    MetaData metaData = MetaData.builder().put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(3).numberOfReplicas(1)).build();
    RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metaData.index("test")).build();
    ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(initialRoutingTable).build();
    assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(3));
    for (int i = 0; i < clusterState.routingTable().index("test").shards().size(); i++) {
        assertThat(clusterState.routingTable().index("test").shard(i).size(), equalTo(2));
        assertThat(clusterState.routingTable().index("test").shard(i).shards().size(), equalTo(2));
        assertThat(clusterState.routingTable().index("test").shard(i).shards().get(0).state(), equalTo(UNASSIGNED));
        assertThat(clusterState.routingTable().index("test").shard(i).shards().get(1).state(), equalTo(UNASSIGNED));
        assertThat(clusterState.routingTable().index("test").shard(i).shards().get(0).currentNodeId(), nullValue());
        assertThat(clusterState.routingTable().index("test").shard(i).shards().get(1).currentNodeId(), nullValue());
    }
    logger.info("Adding three node and performing rerouting");
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2")).add(newNode("node3"))).build();
    ClusterState newState = strategy.reroute(clusterState, "reroute");
    assertThat(newState, not(equalTo(clusterState)));
    clusterState = newState;
    assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(3));
    for (int i = 0; i < clusterState.routingTable().index("test").shards().size(); i++) {
        assertThat(clusterState.routingTable().index("test").shard(i).size(), equalTo(2));
        assertThat(clusterState.routingTable().index("test").shard(i).shards().size(), equalTo(2));
        assertThat(clusterState.routingTable().index("test").shard(i).primaryShard().state(), equalTo(INITIALIZING));
        assertThat(clusterState.routingTable().index("test").shard(i).replicaShards().size(), equalTo(1));
        assertThat(clusterState.routingTable().index("test").shard(i).replicaShards().get(0).state(), equalTo(UNASSIGNED));
        assertThat(clusterState.routingTable().index("test").shard(i).replicaShards().get(0).currentNodeId(), nullValue());
    }
    logger.info("Another round of rebalancing");
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())).build();
    newState = strategy.reroute(clusterState, "reroute");
    assertThat(newState, equalTo(clusterState));
    RoutingNodes routingNodes = clusterState.getRoutingNodes();
    newState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
    assertThat(newState, not(equalTo(clusterState)));
    clusterState = newState;
    assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(3));
    for (int i = 0; i < clusterState.routingTable().index("test").shards().size(); i++) {
        assertThat(clusterState.routingTable().index("test").shard(i).size(), equalTo(2));
        assertThat(clusterState.routingTable().index("test").shard(i).shards().size(), equalTo(2));
        assertThat(clusterState.routingTable().index("test").shard(i).primaryShard().state(), equalTo(STARTED));
        assertThat(clusterState.routingTable().index("test").shard(i).replicaShards().size(), equalTo(1));
        // backup shards are initializing as well, we make sure that they
        // recover from primary *started* shards in the
        // IndicesClusterStateService
        assertThat(clusterState.routingTable().index("test").shard(i).replicaShards().get(0).state(), equalTo(INITIALIZING));
    }
    logger.info("Reroute, nothing should change");
    newState = strategy.reroute(clusterState, "reroute");
    assertThat(newState, equalTo(clusterState));
    logger.info("Start the more shards");
    routingNodes = clusterState.getRoutingNodes();
    newState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
    assertThat(newState, not(equalTo(clusterState)));
    clusterState = newState;
    routingNodes = clusterState.getRoutingNodes();
    assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(3));
    for (int i = 0; i < clusterState.routingTable().index("test").shards().size(); i++) {
        assertThat(clusterState.routingTable().index("test").shard(i).size(), equalTo(2));
        assertThat(clusterState.routingTable().index("test").shard(i).shards().size(), equalTo(2));
        assertThat(clusterState.routingTable().index("test").shard(i).primaryShard().state(), equalTo(STARTED));
        assertThat(clusterState.routingTable().index("test").shard(i).replicaShards().size(), equalTo(1));
    }
    assertThat(routingNodes.node("node1").numberOfShardsWithState(STARTED), equalTo(2));
    assertThat(routingNodes.node("node2").numberOfShardsWithState(STARTED), equalTo(2));
    assertThat(routingNodes.node("node3").numberOfShardsWithState(STARTED), equalTo(2));
    assertThat(routingNodes.node("node1").shardsWithState("test", STARTED).size(), equalTo(2));
    assertThat(routingNodes.node("node2").shardsWithState("test", STARTED).size(), equalTo(2));
    assertThat(routingNodes.node("node3").shardsWithState("test", STARTED).size(), equalTo(2));
    logger.info("Add new index 3 shards 1 replica");
    MetaData updatedMetaData = MetaData.builder(clusterState.metaData()).put(IndexMetaData.builder("test1").settings(settings(Version.CURRENT).put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 3).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1))).build();
    RoutingTable updatedRoutingTable = RoutingTable.builder(clusterState.routingTable()).addAsNew(updatedMetaData.index("test1")).build();
    clusterState = ClusterState.builder(clusterState).metaData(updatedMetaData).routingTable(updatedRoutingTable).build();
    assertThat(clusterState.routingTable().index("test1").shards().size(), equalTo(3));
    newState = strategy.reroute(clusterState, "reroute");
    assertThat(newState, not(equalTo(clusterState)));
    clusterState = newState;
    assertThat(clusterState.routingTable().index("test1").shards().size(), equalTo(3));
    for (int i = 0; i < clusterState.routingTable().index("test1").shards().size(); i++) {
        assertThat(clusterState.routingTable().index("test1").shard(i).size(), equalTo(2));
        assertThat(clusterState.routingTable().index("test1").shard(i).shards().size(), equalTo(2));
        assertThat(clusterState.routingTable().index("test1").shard(i).primaryShard().state(), equalTo(INITIALIZING));
        assertThat(clusterState.routingTable().index("test1").shard(i).replicaShards().size(), equalTo(1));
        assertThat(clusterState.routingTable().index("test1").shard(i).replicaShards().get(0).state(), equalTo(UNASSIGNED));
        assertThat(clusterState.routingTable().index("test1").shard(i).replicaShards().get(0).currentNodeId(), nullValue());
    }
    logger.info("Another round of rebalancing");
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())).build();
    newState = strategy.reroute(clusterState, "reroute");
    assertThat(newState, equalTo(clusterState));
    routingNodes = clusterState.getRoutingNodes();
    newState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
    assertThat(newState, not(equalTo(clusterState)));
    clusterState = newState;
    assertThat(clusterState.routingTable().index("test1").shards().size(), equalTo(3));
    for (int i = 0; i < clusterState.routingTable().index("test1").shards().size(); i++) {
        assertThat(clusterState.routingTable().index("test1").shard(i).size(), equalTo(2));
        assertThat(clusterState.routingTable().index("test1").shard(i).shards().size(), equalTo(2));
        assertThat(clusterState.routingTable().index("test1").shard(i).primaryShard().state(), equalTo(STARTED));
        assertThat(clusterState.routingTable().index("test1").shard(i).replicaShards().size(), equalTo(1));
        // backup shards are initializing as well, we make sure that they
        // recover from primary *started* shards in the
        // IndicesClusterStateService
        assertThat(clusterState.routingTable().index("test1").shard(i).replicaShards().get(0).state(), equalTo(INITIALIZING));
    }
    logger.info("Reroute, nothing should change");
    newState = strategy.reroute(clusterState, "reroute");
    assertThat(newState, equalTo(clusterState));
    logger.info("Start the more shards");
    routingNodes = clusterState.getRoutingNodes();
    newState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
    assertThat(newState, not(equalTo(clusterState)));
    clusterState = newState;
    routingNodes = clusterState.getRoutingNodes();
    assertThat(clusterState.routingTable().index("test1").shards().size(), equalTo(3));
    for (int i = 0; i < clusterState.routingTable().index("test1").shards().size(); i++) {
        assertThat(clusterState.routingTable().index("test1").shard(i).size(), equalTo(2));
        assertThat(clusterState.routingTable().index("test1").shard(i).shards().size(), equalTo(2));
        assertThat(clusterState.routingTable().index("test1").shard(i).primaryShard().state(), equalTo(STARTED));
        assertThat(clusterState.routingTable().index("test1").shard(i).replicaShards().size(), equalTo(1));
    }
    assertThat(routingNodes.node("node1").numberOfShardsWithState(STARTED), equalTo(4));
    assertThat(routingNodes.node("node2").numberOfShardsWithState(STARTED), equalTo(4));
    assertThat(routingNodes.node("node3").numberOfShardsWithState(STARTED), equalTo(4));
    assertThat(routingNodes.node("node1").shardsWithState("test1", STARTED).size(), equalTo(2));
    assertThat(routingNodes.node("node2").shardsWithState("test1", STARTED).size(), equalTo(2));
    assertThat(routingNodes.node("node3").shardsWithState("test1", STARTED).size(), equalTo(2));
}

Example 93 with Metadata

use of org.elasticsearch.cluster.metadata.Metadata in project elasticsearch by elastic.

the class MaxRetryAllocationDeciderTests method createInitialClusterState.

private ClusterState createInitialClusterState() {
    MetaData.Builder metaBuilder = MetaData.builder();
    metaBuilder.put(IndexMetaData.builder("idx").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0));
    MetaData metaData = metaBuilder.build();
    RoutingTable.Builder routingTableBuilder = RoutingTable.builder();
    routingTableBuilder.addAsNew(metaData.index("idx"));
    RoutingTable routingTable = routingTableBuilder.build();
    ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2"))).build();
    RoutingTable prevRoutingTable = routingTable;
    routingTable = strategy.reroute(clusterState, "reroute", false).routingTable();
    clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
    assertEquals(prevRoutingTable.index("idx").shards().size(), 1);
    assertEquals(prevRoutingTable.index("idx").shard(0).shards().get(0).state(), UNASSIGNED);
    assertEquals(routingTable.index("idx").shards().size(), 1);
    assertEquals(routingTable.index("idx").shard(0).shards().get(0).state(), INITIALIZING);
    return clusterState;
}

Example 94 with Metadata

use of org.elasticsearch.cluster.metadata.Metadata in project elasticsearch by elastic.

the class NodeVersionAllocationDeciderTests method testDoNotAllocateFromPrimary.

public void testDoNotAllocateFromPrimary() {
    AllocationService strategy = createAllocationService(Settings.builder().put("cluster.routing.allocation.node_concurrent_recoveries", 10).put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING.getKey(), "always").put("cluster.routing.allocation.cluster_concurrent_rebalance", -1).build());
    logger.info("Building initial routing table");
    MetaData metaData = MetaData.builder().put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(5).numberOfReplicas(2)).build();
    RoutingTable routingTable = RoutingTable.builder().addAsNew(metaData.index("test")).build();
    RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metaData.index("test")).build();
    ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(initialRoutingTable).build();
    assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(5));
    for (int i = 0; i < clusterState.routingTable().index("test").shards().size(); i++) {
        assertThat(clusterState.routingTable().index("test").shard(i).shards().size(), equalTo(3));
        assertThat(clusterState.routingTable().index("test").shard(i).shards().get(0).state(), equalTo(UNASSIGNED));
        assertThat(clusterState.routingTable().index("test").shard(i).shards().get(1).state(), equalTo(UNASSIGNED));
        assertThat(clusterState.routingTable().index("test").shard(i).shards().get(2).state(), equalTo(UNASSIGNED));
        assertThat(clusterState.routingTable().index("test").shard(i).shards().get(0).currentNodeId(), nullValue());
        assertThat(clusterState.routingTable().index("test").shard(i).shards().get(1).currentNodeId(), nullValue());
        assertThat(clusterState.routingTable().index("test").shard(i).shards().get(2).currentNodeId(), nullValue());
    }
    logger.info("start two nodes and fully start the shards");
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2"))).build();
    clusterState = strategy.reroute(clusterState, "reroute");
    for (int i = 0; i < clusterState.routingTable().index("test").shards().size(); i++) {
        assertThat(clusterState.routingTable().index("test").shard(i).shards().size(), equalTo(3));
        assertThat(clusterState.routingTable().index("test").shard(i).primaryShard().state(), equalTo(INITIALIZING));
        assertThat(clusterState.routingTable().index("test").shard(i).replicaShardsWithState(UNASSIGNED).size(), equalTo(2));
    }
    logger.info("start all the primary shards, replicas will start initializing");
    RoutingNodes routingNodes = clusterState.getRoutingNodes();
    clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
    for (int i = 0; i < clusterState.routingTable().index("test").shards().size(); i++) {
        assertThat(clusterState.routingTable().index("test").shard(i).shards().size(), equalTo(3));
        assertThat(clusterState.routingTable().index("test").shard(i).primaryShard().state(), equalTo(STARTED));
        assertThat(clusterState.routingTable().index("test").shard(i).replicaShardsWithState(INITIALIZING).size(), equalTo(1));
        assertThat(clusterState.routingTable().index("test").shard(i).replicaShardsWithState(UNASSIGNED).size(), equalTo(1));
    }
    routingNodes = clusterState.getRoutingNodes();
    clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
    for (int i = 0; i < clusterState.routingTable().index("test").shards().size(); i++) {
        assertThat(clusterState.routingTable().index("test").shard(i).shards().size(), equalTo(3));
        assertThat(clusterState.routingTable().index("test").shard(i).primaryShard().state(), equalTo(STARTED));
        assertThat(clusterState.routingTable().index("test").shard(i).replicaShardsWithState(STARTED).size(), equalTo(1));
        assertThat(clusterState.routingTable().index("test").shard(i).replicaShardsWithState(UNASSIGNED).size(), equalTo(1));
    }
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).add(newNode("node3", VersionUtils.getPreviousVersion()))).build();
    clusterState = strategy.reroute(clusterState, "reroute");
    for (int i = 0; i < clusterState.routingTable().index("test").shards().size(); i++) {
        assertThat(clusterState.routingTable().index("test").shard(i).shards().size(), equalTo(3));
        assertThat(clusterState.routingTable().index("test").shard(i).primaryShard().state(), equalTo(STARTED));
        assertThat(clusterState.routingTable().index("test").shard(i).replicaShardsWithState(STARTED).size(), equalTo(1));
        assertThat(clusterState.routingTable().index("test").shard(i).replicaShardsWithState(UNASSIGNED).size(), equalTo(1));
    }
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).add(newNode("node4"))).build();
    clusterState = strategy.reroute(clusterState, "reroute");
    for (int i = 0; i < clusterState.routingTable().index("test").shards().size(); i++) {
        assertThat(clusterState.routingTable().index("test").shard(i).shards().size(), equalTo(3));
        assertThat(clusterState.routingTable().index("test").shard(i).primaryShard().state(), equalTo(STARTED));
        assertThat(clusterState.routingTable().index("test").shard(i).replicaShardsWithState(STARTED).size(), equalTo(1));
        assertThat(clusterState.routingTable().index("test").shard(i).replicaShardsWithState(INITIALIZING).size(), equalTo(1));
    }
    routingNodes = clusterState.getRoutingNodes();
    clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
    for (int i = 0; i < clusterState.routingTable().index("test").shards().size(); i++) {
        assertThat(clusterState.routingTable().index("test").shard(i).shards().size(), equalTo(3));
        assertThat(clusterState.routingTable().index("test").shard(i).primaryShard().state(), equalTo(STARTED));
        assertThat(clusterState.routingTable().index("test").shard(i).replicaShardsWithState(STARTED).size(), equalTo(2));
    }
}

Example 95 with Metadata

use of org.elasticsearch.cluster.metadata.Metadata in project elasticsearch by elastic.

the class DedicatedClusterSnapshotRestoreIT method testRestoreCustomMetadata.

public void testRestoreCustomMetadata() throws Exception {
    Path tempDir = randomRepoPath();
    logger.info("--> start node");
    internalCluster().startNode();
    Client client = client();
    createIndex("test-idx");
    logger.info("--> add custom persistent metadata");
    updateClusterState(new ClusterStateUpdater() {

        @Override
        public ClusterState execute(ClusterState currentState) throws Exception {
            ClusterState.Builder builder = ClusterState.builder(currentState);
            MetaData.Builder metadataBuilder = MetaData.builder(currentState.metaData());
            metadataBuilder.putCustom(SnapshottableMetadata.TYPE, new SnapshottableMetadata("before_snapshot_s"));
            metadataBuilder.putCustom(NonSnapshottableMetadata.TYPE, new NonSnapshottableMetadata("before_snapshot_ns"));
            metadataBuilder.putCustom(SnapshottableGatewayMetadata.TYPE, new SnapshottableGatewayMetadata("before_snapshot_s_gw"));
            metadataBuilder.putCustom(NonSnapshottableGatewayMetadata.TYPE, new NonSnapshottableGatewayMetadata("before_snapshot_ns_gw"));
            metadataBuilder.putCustom(SnapshotableGatewayNoApiMetadata.TYPE, new SnapshotableGatewayNoApiMetadata("before_snapshot_s_gw_noapi"));
            builder.metaData(metadataBuilder);
            return builder.build();
        }
    });
    logger.info("--> create repository");
    PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo").setType("fs").setSettings(Settings.builder().put("location", tempDir)).execute().actionGet();
    assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
    logger.info("--> start snapshot");
    CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(true).execute().actionGet();
    assertThat(createSnapshotResponse.getSnapshotInfo().totalShards(), greaterThan(0));
    assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().successfulShards()));
    assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").execute().actionGet().getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS));
    logger.info("--> change custom persistent metadata");
    updateClusterState(new ClusterStateUpdater() {

        @Override
        public ClusterState execute(ClusterState currentState) throws Exception {
            ClusterState.Builder builder = ClusterState.builder(currentState);
            MetaData.Builder metadataBuilder = MetaData.builder(currentState.metaData());
            if (randomBoolean()) {
                metadataBuilder.putCustom(SnapshottableMetadata.TYPE, new SnapshottableMetadata("after_snapshot_s"));
            } else {
                metadataBuilder.removeCustom(SnapshottableMetadata.TYPE);
            }
            metadataBuilder.putCustom(NonSnapshottableMetadata.TYPE, new NonSnapshottableMetadata("after_snapshot_ns"));
            if (randomBoolean()) {
                metadataBuilder.putCustom(SnapshottableGatewayMetadata.TYPE, new SnapshottableGatewayMetadata("after_snapshot_s_gw"));
            } else {
                metadataBuilder.removeCustom(SnapshottableGatewayMetadata.TYPE);
            }
            metadataBuilder.putCustom(NonSnapshottableGatewayMetadata.TYPE, new NonSnapshottableGatewayMetadata("after_snapshot_ns_gw"));
            metadataBuilder.removeCustom(SnapshotableGatewayNoApiMetadata.TYPE);
            builder.metaData(metadataBuilder);
            return builder.build();
        }
    });
    logger.info("--> delete repository");
    assertAcked(client.admin().cluster().prepareDeleteRepository("test-repo"));
    logger.info("--> create repository");
    putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo-2").setType("fs").setSettings(Settings.builder().put("location", tempDir)).execute().actionGet();
    assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
    logger.info("--> restore snapshot");
    client.admin().cluster().prepareRestoreSnapshot("test-repo-2", "test-snap").setRestoreGlobalState(true).setIndices("-*").setWaitForCompletion(true).execute().actionGet();
    logger.info("--> make sure old repository wasn't restored");
    assertThrows(client.admin().cluster().prepareGetRepositories("test-repo"), RepositoryMissingException.class);
    assertThat(client.admin().cluster().prepareGetRepositories("test-repo-2").get().repositories().size(), equalTo(1));
    logger.info("--> check that custom persistent metadata was restored");
    ClusterState clusterState = client.admin().cluster().prepareState().get().getState();
    logger.info("Cluster state: {}", clusterState);
    MetaData metaData = clusterState.getMetaData();
    assertThat(((SnapshottableMetadata) metaData.custom(SnapshottableMetadata.TYPE)).getData(), equalTo("before_snapshot_s"));
    assertThat(((NonSnapshottableMetadata) metaData.custom(NonSnapshottableMetadata.TYPE)).getData(), equalTo("after_snapshot_ns"));
    assertThat(((SnapshottableGatewayMetadata) metaData.custom(SnapshottableGatewayMetadata.TYPE)).getData(), equalTo("before_snapshot_s_gw"));
    assertThat(((NonSnapshottableGatewayMetadata) metaData.custom(NonSnapshottableGatewayMetadata.TYPE)).getData(), equalTo("after_snapshot_ns_gw"));
    logger.info("--> restart all nodes");
    internalCluster().fullRestart();
    ensureYellow();
    logger.info("--> check that gateway-persistent custom metadata survived full cluster restart");
    clusterState = client().admin().cluster().prepareState().get().getState();
    logger.info("Cluster state: {}", clusterState);
    metaData = clusterState.getMetaData();
    assertThat(metaData.custom(SnapshottableMetadata.TYPE), nullValue());
    assertThat(metaData.custom(NonSnapshottableMetadata.TYPE), nullValue());
    assertThat(((SnapshottableGatewayMetadata) metaData.custom(SnapshottableGatewayMetadata.TYPE)).getData(), equalTo("before_snapshot_s_gw"));
    assertThat(((NonSnapshottableGatewayMetadata) metaData.custom(NonSnapshottableGatewayMetadata.TYPE)).getData(), equalTo("after_snapshot_ns_gw"));
    // Shouldn't be returned as part of API response
    assertThat(metaData.custom(SnapshotableGatewayNoApiMetadata.TYPE), nullValue());
    // But should still be in state
    metaData = internalCluster().getInstance(ClusterService.class).state().metaData();
    assertThat(((SnapshotableGatewayNoApiMetadata) metaData.custom(SnapshotableGatewayNoApiMetadata.TYPE)).getData(), equalTo("before_snapshot_s_gw_noapi"));
}