Examples with CreateSnapshotResponse org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse used on opensource projects

Example 36 with CreateSnapshotResponse

use of org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse in project elasticsearch by elastic.

the class AbstractS3SnapshotRestoreTest method testEncryption.

@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch-cloud-aws/issues/211")
public void testEncryption() {
    Client client = client();
    logger.info("-->  creating s3 repository with bucket[{}] and path [{}]", internalCluster().getInstance(Settings.class).get("repositories.s3.bucket"), basePath);
    Settings repositorySettings = Settings.builder().put(S3Repository.Repository.BASE_PATH_SETTING.getKey(), basePath).put(S3Repository.Repository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(1000, 10000)).put(S3Repository.Repository.SERVER_SIDE_ENCRYPTION_SETTING.getKey(), true).build();
    PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo").setType("s3").setSettings(repositorySettings).get();
    assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
    createIndex("test-idx-1", "test-idx-2", "test-idx-3");
    ensureGreen();
    logger.info("--> indexing some data");
    for (int i = 0; i < 100; i++) {
        index("test-idx-1", "doc", Integer.toString(i), "foo", "bar" + i);
        index("test-idx-2", "doc", Integer.toString(i), "foo", "baz" + i);
        index("test-idx-3", "doc", Integer.toString(i), "foo", "baz" + i);
    }
    refresh();
    assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().getTotalHits(), equalTo(100L));
    assertThat(client.prepareSearch("test-idx-2").setSize(0).get().getHits().getTotalHits(), equalTo(100L));
    assertThat(client.prepareSearch("test-idx-3").setSize(0).get().getHits().getTotalHits(), equalTo(100L));
    logger.info("--> snapshot");
    CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx-*", "-test-idx-3").get();
    assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(0));
    assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponse.getSnapshotInfo().totalShards()));
    assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").get().getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS));
    Settings settings = internalCluster().getInstance(Settings.class);
    Settings bucket = settings.getByPrefix("repositories.s3.");
    AmazonS3 s3Client = internalCluster().getInstance(AwsS3Service.class).client(repositorySettings, null, randomBoolean(), null);
    String bucketName = bucket.get("bucket");
    logger.info("--> verify encryption for bucket [{}], prefix [{}]", bucketName, basePath);
    List<S3ObjectSummary> summaries = s3Client.listObjects(bucketName, basePath).getObjectSummaries();
    for (S3ObjectSummary summary : summaries) {
        assertThat(s3Client.getObjectMetadata(bucketName, summary.getKey()).getSSEAlgorithm(), equalTo("AES256"));
    }
    logger.info("--> delete some data");
    for (int i = 0; i < 50; i++) {
        client.prepareDelete("test-idx-1", "doc", Integer.toString(i)).get();
    }
    for (int i = 50; i < 100; i++) {
        client.prepareDelete("test-idx-2", "doc", Integer.toString(i)).get();
    }
    for (int i = 0; i < 100; i += 2) {
        client.prepareDelete("test-idx-3", "doc", Integer.toString(i)).get();
    }
    refresh();
    assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().getTotalHits(), equalTo(50L));
    assertThat(client.prepareSearch("test-idx-2").setSize(0).get().getHits().getTotalHits(), equalTo(50L));
    assertThat(client.prepareSearch("test-idx-3").setSize(0).get().getHits().getTotalHits(), equalTo(50L));
    logger.info("--> close indices");
    client.admin().indices().prepareClose("test-idx-1", "test-idx-2").get();
    logger.info("--> restore all indices from the snapshot");
    RestoreSnapshotResponse restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap").setWaitForCompletion(true).execute().actionGet();
    assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
    ensureGreen();
    assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().getTotalHits(), equalTo(100L));
    assertThat(client.prepareSearch("test-idx-2").setSize(0).get().getHits().getTotalHits(), equalTo(100L));
    assertThat(client.prepareSearch("test-idx-3").setSize(0).get().getHits().getTotalHits(), equalTo(50L));
    // Test restore after index deletion
    logger.info("--> delete indices");
    cluster().wipeIndices("test-idx-1", "test-idx-2");
    logger.info("--> restore one index after deletion");
    restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx-*", "-test-idx-2").execute().actionGet();
    assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
    ensureGreen();
    assertThat(client.prepareSearch("test-idx-1").setSize(0).get().getHits().getTotalHits(), equalTo(100L));
    ClusterState clusterState = client.admin().cluster().prepareState().get().getState();
    assertThat(clusterState.getMetaData().hasIndex("test-idx-1"), equalTo(true));
    assertThat(clusterState.getMetaData().hasIndex("test-idx-2"), equalTo(false));
}

Example 37 with CreateSnapshotResponse

use of org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse in project elasticsearch by elastic.

the class SharedClusterSnapshotRestoreIT method testDataFileFailureDuringSnapshot.

public void testDataFileFailureDuringSnapshot() throws Exception {
    Client client = client();
    logger.info("-->  creating repository");
    assertAcked(client.admin().cluster().preparePutRepository("test-repo").setType("mock").setSettings(Settings.builder().put("location", randomRepoPath()).put("random", randomAsciiOfLength(10)).put("random_data_file_io_exception_rate", 0.3)));
    createIndex("test-idx");
    ensureGreen();
    logger.info("--> indexing some data");
    for (int i = 0; i < 100; i++) {
        index("test-idx", "doc", Integer.toString(i), "foo", "bar" + i);
    }
    refresh();
    assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().getTotalHits(), equalTo(100L));
    logger.info("--> snapshot");
    CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx").get();
    if (createSnapshotResponse.getSnapshotInfo().totalShards() == createSnapshotResponse.getSnapshotInfo().successfulShards()) {
        logger.info("--> no failures");
        // If we are here, that means we didn't have any failures, let's check it
        assertThat(getFailureCount("test-repo"), equalTo(0L));
    } else {
        logger.info("--> some failures");
        assertThat(getFailureCount("test-repo"), greaterThan(0L));
        assertThat(createSnapshotResponse.getSnapshotInfo().shardFailures().size(), greaterThan(0));
        for (SnapshotShardFailure shardFailure : createSnapshotResponse.getSnapshotInfo().shardFailures()) {
            assertThat(shardFailure.nodeId(), notNullValue());
            assertThat(shardFailure.index(), equalTo("test-idx"));
        }
        GetSnapshotsResponse getSnapshotsResponse = client.admin().cluster().prepareGetSnapshots("test-repo").addSnapshots("test-snap").get();
        assertThat(getSnapshotsResponse.getSnapshots().size(), equalTo(1));
        SnapshotInfo snapshotInfo = getSnapshotsResponse.getSnapshots().get(0);
        assertThat(snapshotInfo.state(), equalTo(SnapshotState.PARTIAL));
        assertThat(snapshotInfo.shardFailures().size(), greaterThan(0));
        assertThat(snapshotInfo.totalShards(), greaterThan(snapshotInfo.successfulShards()));
        // Verify that snapshot status also contains the same failures
        SnapshotsStatusResponse snapshotsStatusResponse = client.admin().cluster().prepareSnapshotStatus("test-repo").addSnapshots("test-snap").get();
        assertThat(snapshotsStatusResponse.getSnapshots().size(), equalTo(1));
        SnapshotStatus snapshotStatus = snapshotsStatusResponse.getSnapshots().get(0);
        assertThat(snapshotStatus.getIndices().size(), equalTo(1));
        SnapshotIndexStatus indexStatus = snapshotStatus.getIndices().get("test-idx");
        assertThat(indexStatus, notNullValue());
        assertThat(indexStatus.getShardsStats().getFailedShards(), equalTo(snapshotInfo.failedShards()));
        assertThat(indexStatus.getShardsStats().getDoneShards(), equalTo(snapshotInfo.successfulShards()));
        assertThat(indexStatus.getShards().size(), equalTo(snapshotInfo.totalShards()));
        int numberOfFailures = 0;
        for (SnapshotIndexShardStatus shardStatus : indexStatus.getShards().values()) {
            if (shardStatus.getStage() == SnapshotIndexShardStage.FAILURE) {
                assertThat(shardStatus.getFailure(), notNullValue());
                numberOfFailures++;
            } else {
                assertThat(shardStatus.getFailure(), nullValue());
            }
        }
        assertThat(indexStatus.getShardsStats().getFailedShards(), equalTo(numberOfFailures));
    }
}

Example 38 with CreateSnapshotResponse

use of org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse in project elasticsearch by elastic.

the class SharedClusterSnapshotRestoreIT method testDeletionOfFailingToRecoverIndexShouldStopRestore.

public void testDeletionOfFailingToRecoverIndexShouldStopRestore() throws Exception {
    Path repositoryLocation = randomRepoPath();
    Client client = client();
    logger.info("-->  creating repository");
    assertAcked(client.admin().cluster().preparePutRepository("test-repo").setType("fs").setSettings(Settings.builder().put("location", repositoryLocation)));
    createIndex("test-idx");
    ensureGreen();
    logger.info("--> indexing some data");
    for (int i = 0; i < 100; i++) {
        index("test-idx", "doc", Integer.toString(i), "foo", "bar" + i);
    }
    refresh();
    assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().getTotalHits(), equalTo(100L));
    logger.info("--> snapshot");
    CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx").get();
    assertThat(createSnapshotResponse.getSnapshotInfo().state(), equalTo(SnapshotState.SUCCESS));
    assertThat(createSnapshotResponse.getSnapshotInfo().totalShards(), equalTo(createSnapshotResponse.getSnapshotInfo().successfulShards()));
    logger.info("-->  update repository with mock version");
    assertAcked(client.admin().cluster().preparePutRepository("test-repo").setType("mock").setSettings(Settings.builder().put("location", repositoryLocation).put("random", randomAsciiOfLength(10)).put("random_data_file_io_exception_rate", // Fail completely
    1.0)));
    // Test restore after index deletion
    logger.info("--> delete index");
    cluster().wipeIndices("test-idx");
    logger.info("--> restore index after deletion");
    ListenableActionFuture<RestoreSnapshotResponse> restoreSnapshotResponseFuture = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap").setWaitForCompletion(true).execute();
    logger.info("--> wait for the index to appear");
    //  that would mean that recovery process started and failing
    assertThat(waitForIndex("test-idx", TimeValue.timeValueSeconds(10)), equalTo(true));
    logger.info("--> delete index");
    cluster().wipeIndices("test-idx");
    logger.info("--> get restore results");
    // Now read restore results and make sure it failed
    RestoreSnapshotResponse restoreSnapshotResponse = restoreSnapshotResponseFuture.actionGet(TimeValue.timeValueSeconds(10));
    assertThat(restoreSnapshotResponse.getRestoreInfo().failedShards(), greaterThan(0));
    assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), equalTo(restoreSnapshotResponse.getRestoreInfo().failedShards()));
    logger.info("--> restoring working repository");
    assertAcked(client.admin().cluster().preparePutRepository("test-repo").setType("fs").setSettings(Settings.builder().put("location", repositoryLocation)));
    logger.info("--> trying to restore index again");
    restoreSnapshotResponse = client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap").setWaitForCompletion(true).execute().actionGet();
    assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
    assertThat(restoreSnapshotResponse.getRestoreInfo().failedShards(), equalTo(0));
    SearchResponse countResponse = client.prepareSearch("test-idx").setSize(0).get();
    assertThat(countResponse.getHits().getTotalHits(), equalTo(100L));
}

Example 39 with CreateSnapshotResponse

use of org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse in project elasticsearch by elastic.

the class DedicatedClusterSnapshotRestoreIT method testRestoreIndexWithMissingShards.

public void testRestoreIndexWithMissingShards() throws Exception {
    logger.info("--> start 2 nodes");
    internalCluster().startNode();
    internalCluster().startNode();
    cluster().wipeIndices("_all");
    logger.info("--> create an index that will have some unallocated shards");
    assertAcked(prepareCreate("test-idx-some", 2, Settings.builder().put("number_of_shards", 6).put("number_of_replicas", 0)));
    ensureGreen();
    logger.info("--> indexing some data into test-idx-some");
    for (int i = 0; i < 100; i++) {
        index("test-idx-some", "doc", Integer.toString(i), "foo", "bar" + i);
    }
    refresh();
    assertThat(client().prepareSearch("test-idx-some").setSize(0).get().getHits().getTotalHits(), equalTo(100L));
    logger.info("--> shutdown one of the nodes");
    internalCluster().stopRandomDataNode();
    assertThat(client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setTimeout("1m").setWaitForNodes("<2").execute().actionGet().isTimedOut(), equalTo(false));
    logger.info("--> create an index that will have all allocated shards");
    assertAcked(prepareCreate("test-idx-all", 1, Settings.builder().put("number_of_shards", 6).put("number_of_replicas", 0)));
    ensureGreen("test-idx-all");
    logger.info("--> create an index that will be closed");
    assertAcked(prepareCreate("test-idx-closed", 1, Settings.builder().put("number_of_shards", 4).put("number_of_replicas", 0)));
    ensureGreen("test-idx-closed");
    logger.info("--> indexing some data into test-idx-all");
    for (int i = 0; i < 100; i++) {
        index("test-idx-all", "doc", Integer.toString(i), "foo", "bar" + i);
        index("test-idx-closed", "doc", Integer.toString(i), "foo", "bar" + i);
    }
    // don't refresh test-idx-some it will take 30 sec until it times out...
    refresh("test-idx-closed", "test-idx-all");
    assertThat(client().prepareSearch("test-idx-all").setSize(0).get().getHits().getTotalHits(), equalTo(100L));
    assertAcked(client().admin().indices().prepareClose("test-idx-closed"));
    logger.info("--> create an index that will have no allocated shards");
    assertAcked(prepareCreate("test-idx-none", 1, Settings.builder().put("number_of_shards", 6).put("index.routing.allocation.include.tag", "nowhere").put("number_of_replicas", 0)).setWaitForActiveShards(ActiveShardCount.NONE).get());
    assertTrue(client().admin().indices().prepareExists("test-idx-none").get().isExists());
    logger.info("--> creating repository");
    PutRepositoryResponse putRepositoryResponse = client().admin().cluster().preparePutRepository("test-repo").setType("fs").setSettings(Settings.builder().put("location", randomRepoPath())).execute().actionGet();
    assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
    logger.info("--> start snapshot with default settings and closed index - should be blocked");
    assertBlocked(client().admin().cluster().prepareCreateSnapshot("test-repo", "test-snap-1").setIndices("test-idx-all", "test-idx-none", "test-idx-some", "test-idx-closed").setWaitForCompletion(true), MetaDataIndexStateService.INDEX_CLOSED_BLOCK);
    logger.info("--> start snapshot with default settings without a closed index - should fail");
    CreateSnapshotResponse createSnapshotResponse = client().admin().cluster().prepareCreateSnapshot("test-repo", "test-snap-1").setIndices("test-idx-all", "test-idx-none", "test-idx-some").setWaitForCompletion(true).execute().actionGet();
    assertThat(createSnapshotResponse.getSnapshotInfo().state(), equalTo(SnapshotState.FAILED));
    assertThat(createSnapshotResponse.getSnapshotInfo().reason(), containsString("Indices don't have primary shards"));
    if (randomBoolean()) {
        logger.info("checking snapshot completion using status");
        client().admin().cluster().prepareCreateSnapshot("test-repo", "test-snap-2").setIndices("test-idx-all", "test-idx-none", "test-idx-some").setWaitForCompletion(false).setPartial(true).execute().actionGet();
        assertBusy(new Runnable() {

            @Override
            public void run() {
                SnapshotsStatusResponse snapshotsStatusResponse = client().admin().cluster().prepareSnapshotStatus("test-repo").setSnapshots("test-snap-2").get();
                List<SnapshotStatus> snapshotStatuses = snapshotsStatusResponse.getSnapshots();
                assertEquals(snapshotStatuses.size(), 1);
                logger.trace("current snapshot status [{}]", snapshotStatuses.get(0));
                assertTrue(snapshotStatuses.get(0).getState().completed());
            }
        }, 1, TimeUnit.MINUTES);
        SnapshotsStatusResponse snapshotsStatusResponse = client().admin().cluster().prepareSnapshotStatus("test-repo").setSnapshots("test-snap-2").get();
        List<SnapshotStatus> snapshotStatuses = snapshotsStatusResponse.getSnapshots();
        assertThat(snapshotStatuses.size(), equalTo(1));
        SnapshotStatus snapshotStatus = snapshotStatuses.get(0);
        logger.info("State: [{}], Reason: [{}]", createSnapshotResponse.getSnapshotInfo().state(), createSnapshotResponse.getSnapshotInfo().reason());
        assertThat(snapshotStatus.getShardsStats().getTotalShards(), equalTo(18));
        assertThat(snapshotStatus.getShardsStats().getDoneShards(), lessThan(12));
        assertThat(snapshotStatus.getShardsStats().getDoneShards(), greaterThan(6));
        // There is slight delay between snapshot being marked as completed in the cluster state and on the file system
        // After it was marked as completed in the cluster state - we need to check if it's completed on the file system as well
        assertBusy(new Runnable() {

            @Override
            public void run() {
                GetSnapshotsResponse response = client().admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap-2").get();
                assertThat(response.getSnapshots().size(), equalTo(1));
                SnapshotInfo snapshotInfo = response.getSnapshots().get(0);
                assertTrue(snapshotInfo.state().completed());
                assertEquals(SnapshotState.PARTIAL, snapshotInfo.state());
            }
        }, 1, TimeUnit.MINUTES);
    } else {
        logger.info("checking snapshot completion using wait_for_completion flag");
        createSnapshotResponse = client().admin().cluster().prepareCreateSnapshot("test-repo", "test-snap-2").setIndices("test-idx-all", "test-idx-none", "test-idx-some").setWaitForCompletion(true).setPartial(true).execute().actionGet();
        logger.info("State: [{}], Reason: [{}]", createSnapshotResponse.getSnapshotInfo().state(), createSnapshotResponse.getSnapshotInfo().reason());
        assertThat(createSnapshotResponse.getSnapshotInfo().totalShards(), equalTo(18));
        assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), lessThan(12));
        assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(6));
        assertThat(client().admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap-2").execute().actionGet().getSnapshots().get(0).state(), equalTo(SnapshotState.PARTIAL));
    }
    assertAcked(client().admin().indices().prepareClose("test-idx-some", "test-idx-all").execute().actionGet());
    logger.info("--> restore incomplete snapshot - should fail");
    assertThrows(client().admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap-2").setRestoreGlobalState(false).setWaitForCompletion(true).execute(), SnapshotRestoreException.class);
    logger.info("--> restore snapshot for the index that was snapshotted completely");
    RestoreSnapshotResponse restoreSnapshotResponse = client().admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap-2").setRestoreGlobalState(false).setIndices("test-idx-all").setWaitForCompletion(true).execute().actionGet();
    assertThat(restoreSnapshotResponse.getRestoreInfo(), notNullValue());
    assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), equalTo(6));
    assertThat(restoreSnapshotResponse.getRestoreInfo().successfulShards(), equalTo(6));
    assertThat(restoreSnapshotResponse.getRestoreInfo().failedShards(), equalTo(0));
    assertThat(client().prepareSearch("test-idx-all").setSize(0).get().getHits().getTotalHits(), equalTo(100L));
    logger.info("--> restore snapshot for the partial index");
    cluster().wipeIndices("test-idx-some");
    restoreSnapshotResponse = client().admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap-2").setRestoreGlobalState(false).setIndices("test-idx-some").setPartial(true).setWaitForCompletion(true).get();
    assertThat(restoreSnapshotResponse.getRestoreInfo(), notNullValue());
    assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), equalTo(6));
    assertThat(restoreSnapshotResponse.getRestoreInfo().successfulShards(), allOf(greaterThan(0), lessThan(6)));
    assertThat(restoreSnapshotResponse.getRestoreInfo().failedShards(), greaterThan(0));
    assertThat(client().prepareSearch("test-idx-some").setSize(0).get().getHits().getTotalHits(), allOf(greaterThan(0L), lessThan(100L)));
    logger.info("--> restore snapshot for the index that didn't have any shards snapshotted successfully");
    cluster().wipeIndices("test-idx-none");
    restoreSnapshotResponse = client().admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap-2").setRestoreGlobalState(false).setIndices("test-idx-none").setPartial(true).setWaitForCompletion(true).get();
    assertThat(restoreSnapshotResponse.getRestoreInfo(), notNullValue());
    assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), equalTo(6));
    assertThat(restoreSnapshotResponse.getRestoreInfo().successfulShards(), equalTo(0));
    assertThat(restoreSnapshotResponse.getRestoreInfo().failedShards(), equalTo(6));
    assertThat(client().prepareSearch("test-idx-some").setSize(0).get().getHits().getTotalHits(), allOf(greaterThan(0L), lessThan(100L)));
}

Example 40 with CreateSnapshotResponse

use of org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse in project elasticsearch by elastic.

the class DedicatedClusterSnapshotRestoreIT method testRestorePersistentSettings.

public void testRestorePersistentSettings() throws Exception {
    logger.info("--> start 2 nodes");
    Settings nodeSettings = Settings.builder().put("discovery.type", "zen").put(ZenDiscovery.PING_TIMEOUT_SETTING.getKey(), "200ms").put("discovery.initial_state_timeout", "500ms").build();
    internalCluster().startNode(nodeSettings);
    Client client = client();
    String secondNode = internalCluster().startNode(nodeSettings);
    logger.info("--> wait for the second node to join the cluster");
    assertThat(client.admin().cluster().prepareHealth().setWaitForNodes("2").get().isTimedOut(), equalTo(false));
    int random = randomIntBetween(10, 42);
    logger.info("--> set test persistent setting");
    client.admin().cluster().prepareUpdateSettings().setPersistentSettings(Settings.builder().put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), 2)).execute().actionGet();
    assertThat(client.admin().cluster().prepareState().setRoutingTable(false).setNodes(false).execute().actionGet().getState().getMetaData().persistentSettings().getAsInt(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), -1), equalTo(2));
    logger.info("--> create repository");
    PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo").setType("fs").setSettings(Settings.builder().put("location", randomRepoPath())).execute().actionGet();
    assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
    logger.info("--> start snapshot");
    CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(true).execute().actionGet();
    assertThat(createSnapshotResponse.getSnapshotInfo().totalShards(), equalTo(0));
    assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(0));
    assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").execute().actionGet().getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS));
    logger.info("--> clean the test persistent setting");
    client.admin().cluster().prepareUpdateSettings().setPersistentSettings(Settings.builder().put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), 1)).execute().actionGet();
    assertThat(client.admin().cluster().prepareState().setRoutingTable(false).setNodes(false).execute().actionGet().getState().getMetaData().persistentSettings().getAsInt(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), -1), equalTo(1));
    stopNode(secondNode);
    assertThat(client.admin().cluster().prepareHealth().setWaitForNodes("1").get().isTimedOut(), equalTo(false));
    logger.info("--> restore snapshot");
    try {
        client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap").setRestoreGlobalState(true).setWaitForCompletion(true).execute().actionGet();
        fail("can't restore minimum master nodes");
    } catch (IllegalArgumentException ex) {
        assertEquals("illegal value can't update [discovery.zen.minimum_master_nodes] from [1] to [2]", ex.getMessage());
        assertEquals("cannot set discovery.zen.minimum_master_nodes to more than the current master nodes count [1]", ex.getCause().getMessage());
    }
    logger.info("--> ensure that zen discovery minimum master nodes wasn't restored");
    assertThat(client.admin().cluster().prepareState().setRoutingTable(false).setNodes(false).execute().actionGet().getState().getMetaData().persistentSettings().getAsInt(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), -1), not(equalTo(2)));
}