Examples with PutRepositoryResponse org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryResponse used on opensource projects

Example 21 with PutRepositoryResponse

use of org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryResponse in project elasticsearch by elastic.

the class DedicatedClusterSnapshotRestoreIT method testSnapshotDuringNodeShutdown.

public void testSnapshotDuringNodeShutdown() throws Exception {
    logger.info("--> start 2 nodes");
    Client client = client();
    assertAcked(prepareCreate("test-idx", 2, Settings.builder().put("number_of_shards", 2).put("number_of_replicas", 0)));
    ensureGreen();
    logger.info("--> indexing some data");
    for (int i = 0; i < 100; i++) {
        index("test-idx", "doc", Integer.toString(i), "foo", "bar" + i);
    }
    refresh();
    assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().getTotalHits(), equalTo(100L));
    logger.info("--> create repository");
    logger.info("--> creating repository");
    PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo").setType("mock").setSettings(Settings.builder().put("location", randomRepoPath()).put("random", randomAsciiOfLength(10)).put("wait_after_unblock", 200)).get();
    assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
    // Pick one node and block it
    String blockedNode = blockNodeWithIndex("test-repo", "test-idx");
    logger.info("--> snapshot");
    client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(false).setIndices("test-idx").get();
    logger.info("--> waiting for block to kick in");
    waitForBlock(blockedNode, "test-repo", TimeValue.timeValueSeconds(60));
    logger.info("--> execution was blocked on node [{}], shutting it down", blockedNode);
    unblockNode("test-repo", blockedNode);
    logger.info("--> stopping node [{}]", blockedNode);
    stopNode(blockedNode);
    logger.info("--> waiting for completion");
    SnapshotInfo snapshotInfo = waitForCompletion("test-repo", "test-snap", TimeValue.timeValueSeconds(60));
    logger.info("Number of failed shards [{}]", snapshotInfo.shardFailures().size());
    logger.info("--> done");
}

Example 22 with PutRepositoryResponse

use of org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryResponse in project elasticsearch by elastic.

the class DedicatedClusterSnapshotRestoreIT method testRestoreIndexWithMissingShards.

public void testRestoreIndexWithMissingShards() throws Exception {
    logger.info("--> start 2 nodes");
    internalCluster().startNode();
    internalCluster().startNode();
    cluster().wipeIndices("_all");
    logger.info("--> create an index that will have some unallocated shards");
    assertAcked(prepareCreate("test-idx-some", 2, Settings.builder().put("number_of_shards", 6).put("number_of_replicas", 0)));
    ensureGreen();
    logger.info("--> indexing some data into test-idx-some");
    for (int i = 0; i < 100; i++) {
        index("test-idx-some", "doc", Integer.toString(i), "foo", "bar" + i);
    }
    refresh();
    assertThat(client().prepareSearch("test-idx-some").setSize(0).get().getHits().getTotalHits(), equalTo(100L));
    logger.info("--> shutdown one of the nodes");
    internalCluster().stopRandomDataNode();
    assertThat(client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setTimeout("1m").setWaitForNodes("<2").execute().actionGet().isTimedOut(), equalTo(false));
    logger.info("--> create an index that will have all allocated shards");
    assertAcked(prepareCreate("test-idx-all", 1, Settings.builder().put("number_of_shards", 6).put("number_of_replicas", 0)));
    ensureGreen("test-idx-all");
    logger.info("--> create an index that will be closed");
    assertAcked(prepareCreate("test-idx-closed", 1, Settings.builder().put("number_of_shards", 4).put("number_of_replicas", 0)));
    ensureGreen("test-idx-closed");
    logger.info("--> indexing some data into test-idx-all");
    for (int i = 0; i < 100; i++) {
        index("test-idx-all", "doc", Integer.toString(i), "foo", "bar" + i);
        index("test-idx-closed", "doc", Integer.toString(i), "foo", "bar" + i);
    }
    // don't refresh test-idx-some it will take 30 sec until it times out...
    refresh("test-idx-closed", "test-idx-all");
    assertThat(client().prepareSearch("test-idx-all").setSize(0).get().getHits().getTotalHits(), equalTo(100L));
    assertAcked(client().admin().indices().prepareClose("test-idx-closed"));
    logger.info("--> create an index that will have no allocated shards");
    assertAcked(prepareCreate("test-idx-none", 1, Settings.builder().put("number_of_shards", 6).put("index.routing.allocation.include.tag", "nowhere").put("number_of_replicas", 0)).setWaitForActiveShards(ActiveShardCount.NONE).get());
    assertTrue(client().admin().indices().prepareExists("test-idx-none").get().isExists());
    logger.info("--> creating repository");
    PutRepositoryResponse putRepositoryResponse = client().admin().cluster().preparePutRepository("test-repo").setType("fs").setSettings(Settings.builder().put("location", randomRepoPath())).execute().actionGet();
    assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
    logger.info("--> start snapshot with default settings and closed index - should be blocked");
    assertBlocked(client().admin().cluster().prepareCreateSnapshot("test-repo", "test-snap-1").setIndices("test-idx-all", "test-idx-none", "test-idx-some", "test-idx-closed").setWaitForCompletion(true), MetaDataIndexStateService.INDEX_CLOSED_BLOCK);
    logger.info("--> start snapshot with default settings without a closed index - should fail");
    CreateSnapshotResponse createSnapshotResponse = client().admin().cluster().prepareCreateSnapshot("test-repo", "test-snap-1").setIndices("test-idx-all", "test-idx-none", "test-idx-some").setWaitForCompletion(true).execute().actionGet();
    assertThat(createSnapshotResponse.getSnapshotInfo().state(), equalTo(SnapshotState.FAILED));
    assertThat(createSnapshotResponse.getSnapshotInfo().reason(), containsString("Indices don't have primary shards"));
    if (randomBoolean()) {
        logger.info("checking snapshot completion using status");
        client().admin().cluster().prepareCreateSnapshot("test-repo", "test-snap-2").setIndices("test-idx-all", "test-idx-none", "test-idx-some").setWaitForCompletion(false).setPartial(true).execute().actionGet();
        assertBusy(new Runnable() {

            @Override
            public void run() {
                SnapshotsStatusResponse snapshotsStatusResponse = client().admin().cluster().prepareSnapshotStatus("test-repo").setSnapshots("test-snap-2").get();
                List<SnapshotStatus> snapshotStatuses = snapshotsStatusResponse.getSnapshots();
                assertEquals(snapshotStatuses.size(), 1);
                logger.trace("current snapshot status [{}]", snapshotStatuses.get(0));
                assertTrue(snapshotStatuses.get(0).getState().completed());
            }
        }, 1, TimeUnit.MINUTES);
        SnapshotsStatusResponse snapshotsStatusResponse = client().admin().cluster().prepareSnapshotStatus("test-repo").setSnapshots("test-snap-2").get();
        List<SnapshotStatus> snapshotStatuses = snapshotsStatusResponse.getSnapshots();
        assertThat(snapshotStatuses.size(), equalTo(1));
        SnapshotStatus snapshotStatus = snapshotStatuses.get(0);
        logger.info("State: [{}], Reason: [{}]", createSnapshotResponse.getSnapshotInfo().state(), createSnapshotResponse.getSnapshotInfo().reason());
        assertThat(snapshotStatus.getShardsStats().getTotalShards(), equalTo(18));
        assertThat(snapshotStatus.getShardsStats().getDoneShards(), lessThan(12));
        assertThat(snapshotStatus.getShardsStats().getDoneShards(), greaterThan(6));
        // There is slight delay between snapshot being marked as completed in the cluster state and on the file system
        // After it was marked as completed in the cluster state - we need to check if it's completed on the file system as well
        assertBusy(new Runnable() {

            @Override
            public void run() {
                GetSnapshotsResponse response = client().admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap-2").get();
                assertThat(response.getSnapshots().size(), equalTo(1));
                SnapshotInfo snapshotInfo = response.getSnapshots().get(0);
                assertTrue(snapshotInfo.state().completed());
                assertEquals(SnapshotState.PARTIAL, snapshotInfo.state());
            }
        }, 1, TimeUnit.MINUTES);
    } else {
        logger.info("checking snapshot completion using wait_for_completion flag");
        createSnapshotResponse = client().admin().cluster().prepareCreateSnapshot("test-repo", "test-snap-2").setIndices("test-idx-all", "test-idx-none", "test-idx-some").setWaitForCompletion(true).setPartial(true).execute().actionGet();
        logger.info("State: [{}], Reason: [{}]", createSnapshotResponse.getSnapshotInfo().state(), createSnapshotResponse.getSnapshotInfo().reason());
        assertThat(createSnapshotResponse.getSnapshotInfo().totalShards(), equalTo(18));
        assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), lessThan(12));
        assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), greaterThan(6));
        assertThat(client().admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap-2").execute().actionGet().getSnapshots().get(0).state(), equalTo(SnapshotState.PARTIAL));
    }
    assertAcked(client().admin().indices().prepareClose("test-idx-some", "test-idx-all").execute().actionGet());
    logger.info("--> restore incomplete snapshot - should fail");
    assertThrows(client().admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap-2").setRestoreGlobalState(false).setWaitForCompletion(true).execute(), SnapshotRestoreException.class);
    logger.info("--> restore snapshot for the index that was snapshotted completely");
    RestoreSnapshotResponse restoreSnapshotResponse = client().admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap-2").setRestoreGlobalState(false).setIndices("test-idx-all").setWaitForCompletion(true).execute().actionGet();
    assertThat(restoreSnapshotResponse.getRestoreInfo(), notNullValue());
    assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), equalTo(6));
    assertThat(restoreSnapshotResponse.getRestoreInfo().successfulShards(), equalTo(6));
    assertThat(restoreSnapshotResponse.getRestoreInfo().failedShards(), equalTo(0));
    assertThat(client().prepareSearch("test-idx-all").setSize(0).get().getHits().getTotalHits(), equalTo(100L));
    logger.info("--> restore snapshot for the partial index");
    cluster().wipeIndices("test-idx-some");
    restoreSnapshotResponse = client().admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap-2").setRestoreGlobalState(false).setIndices("test-idx-some").setPartial(true).setWaitForCompletion(true).get();
    assertThat(restoreSnapshotResponse.getRestoreInfo(), notNullValue());
    assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), equalTo(6));
    assertThat(restoreSnapshotResponse.getRestoreInfo().successfulShards(), allOf(greaterThan(0), lessThan(6)));
    assertThat(restoreSnapshotResponse.getRestoreInfo().failedShards(), greaterThan(0));
    assertThat(client().prepareSearch("test-idx-some").setSize(0).get().getHits().getTotalHits(), allOf(greaterThan(0L), lessThan(100L)));
    logger.info("--> restore snapshot for the index that didn't have any shards snapshotted successfully");
    cluster().wipeIndices("test-idx-none");
    restoreSnapshotResponse = client().admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap-2").setRestoreGlobalState(false).setIndices("test-idx-none").setPartial(true).setWaitForCompletion(true).get();
    assertThat(restoreSnapshotResponse.getRestoreInfo(), notNullValue());
    assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), equalTo(6));
    assertThat(restoreSnapshotResponse.getRestoreInfo().successfulShards(), equalTo(0));
    assertThat(restoreSnapshotResponse.getRestoreInfo().failedShards(), equalTo(6));
    assertThat(client().prepareSearch("test-idx-some").setSize(0).get().getHits().getTotalHits(), allOf(greaterThan(0L), lessThan(100L)));
}

Example 23 with PutRepositoryResponse

use of org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryResponse in project elasticsearch by elastic.

the class DedicatedClusterSnapshotRestoreIT method testRestorePersistentSettings.

public void testRestorePersistentSettings() throws Exception {
    logger.info("--> start 2 nodes");
    Settings nodeSettings = Settings.builder().put("discovery.type", "zen").put(ZenDiscovery.PING_TIMEOUT_SETTING.getKey(), "200ms").put("discovery.initial_state_timeout", "500ms").build();
    internalCluster().startNode(nodeSettings);
    Client client = client();
    String secondNode = internalCluster().startNode(nodeSettings);
    logger.info("--> wait for the second node to join the cluster");
    assertThat(client.admin().cluster().prepareHealth().setWaitForNodes("2").get().isTimedOut(), equalTo(false));
    int random = randomIntBetween(10, 42);
    logger.info("--> set test persistent setting");
    client.admin().cluster().prepareUpdateSettings().setPersistentSettings(Settings.builder().put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), 2)).execute().actionGet();
    assertThat(client.admin().cluster().prepareState().setRoutingTable(false).setNodes(false).execute().actionGet().getState().getMetaData().persistentSettings().getAsInt(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), -1), equalTo(2));
    logger.info("--> create repository");
    PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo").setType("fs").setSettings(Settings.builder().put("location", randomRepoPath())).execute().actionGet();
    assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
    logger.info("--> start snapshot");
    CreateSnapshotResponse createSnapshotResponse = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(true).execute().actionGet();
    assertThat(createSnapshotResponse.getSnapshotInfo().totalShards(), equalTo(0));
    assertThat(createSnapshotResponse.getSnapshotInfo().successfulShards(), equalTo(0));
    assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").execute().actionGet().getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS));
    logger.info("--> clean the test persistent setting");
    client.admin().cluster().prepareUpdateSettings().setPersistentSettings(Settings.builder().put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), 1)).execute().actionGet();
    assertThat(client.admin().cluster().prepareState().setRoutingTable(false).setNodes(false).execute().actionGet().getState().getMetaData().persistentSettings().getAsInt(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), -1), equalTo(1));
    stopNode(secondNode);
    assertThat(client.admin().cluster().prepareHealth().setWaitForNodes("1").get().isTimedOut(), equalTo(false));
    logger.info("--> restore snapshot");
    try {
        client.admin().cluster().prepareRestoreSnapshot("test-repo", "test-snap").setRestoreGlobalState(true).setWaitForCompletion(true).execute().actionGet();
        fail("can't restore minimum master nodes");
    } catch (IllegalArgumentException ex) {
        assertEquals("illegal value can't update [discovery.zen.minimum_master_nodes] from [1] to [2]", ex.getMessage());
        assertEquals("cannot set discovery.zen.minimum_master_nodes to more than the current master nodes count [1]", ex.getCause().getMessage());
    }
    logger.info("--> ensure that zen discovery minimum master nodes wasn't restored");
    assertThat(client.admin().cluster().prepareState().setRoutingTable(false).setNodes(false).execute().actionGet().getState().getMetaData().persistentSettings().getAsInt(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), -1), not(equalTo(2)));
}

Example 24 with PutRepositoryResponse

use of org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryResponse in project elasticsearch by elastic.

the class SharedClusterSnapshotRestoreIT method testSnapshotStatus.

public void testSnapshotStatus() throws Exception {
    Client client = client();
    Path repositoryLocation = randomRepoPath();
    logger.info("-->  creating repository");
    PutRepositoryResponse putRepositoryResponse = client.admin().cluster().preparePutRepository("test-repo").setType("mock").setSettings(Settings.builder().put("location", repositoryLocation).put("random", randomAsciiOfLength(10)).put("wait_after_unblock", 200)).get();
    assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
    // Create index on 2 nodes and make sure each node has a primary by setting no replicas
    assertAcked(prepareCreate("test-idx", 2, Settings.builder().put("number_of_replicas", 0)));
    logger.info("--> indexing some data");
    for (int i = 0; i < 100; i++) {
        index("test-idx", "doc", Integer.toString(i), "foo", "bar" + i);
    }
    refresh();
    assertThat(client.prepareSearch("test-idx").setSize(0).get().getHits().getTotalHits(), equalTo(100L));
    // Pick one node and block it
    String blockedNode = blockNodeWithIndex("test-repo", "test-idx");
    logger.info("--> snapshot");
    client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(false).setIndices("test-idx").get();
    logger.info("--> waiting for block to kick in");
    waitForBlock(blockedNode, "test-repo", TimeValue.timeValueSeconds(60));
    logger.info("--> execution was blocked on node [{}], checking snapshot status with specified repository and snapshot", blockedNode);
    SnapshotsStatusResponse response = client.admin().cluster().prepareSnapshotStatus("test-repo").execute().actionGet();
    assertThat(response.getSnapshots().size(), equalTo(1));
    SnapshotStatus snapshotStatus = response.getSnapshots().get(0);
    assertThat(snapshotStatus.getState(), equalTo(SnapshotsInProgress.State.STARTED));
    // We blocked the node during data write operation, so at least one shard snapshot should be in STARTED stage
    assertThat(snapshotStatus.getShardsStats().getStartedShards(), greaterThan(0));
    for (SnapshotIndexShardStatus shardStatus : snapshotStatus.getIndices().get("test-idx")) {
        if (shardStatus.getStage() == SnapshotIndexShardStage.STARTED) {
            assertThat(shardStatus.getNodeId(), notNullValue());
        }
    }
    logger.info("--> checking snapshot status for all currently running and snapshot with empty repository");
    response = client.admin().cluster().prepareSnapshotStatus().execute().actionGet();
    assertThat(response.getSnapshots().size(), equalTo(1));
    snapshotStatus = response.getSnapshots().get(0);
    assertThat(snapshotStatus.getState(), equalTo(SnapshotsInProgress.State.STARTED));
    // We blocked the node during data write operation, so at least one shard snapshot should be in STARTED stage
    assertThat(snapshotStatus.getShardsStats().getStartedShards(), greaterThan(0));
    for (SnapshotIndexShardStatus shardStatus : snapshotStatus.getIndices().get("test-idx")) {
        if (shardStatus.getStage() == SnapshotIndexShardStage.STARTED) {
            assertThat(shardStatus.getNodeId(), notNullValue());
        }
    }
    logger.info("--> checking that _current returns the currently running snapshot");
    GetSnapshotsResponse getResponse = client.admin().cluster().prepareGetSnapshots("test-repo").setCurrentSnapshot().execute().actionGet();
    assertThat(getResponse.getSnapshots().size(), equalTo(1));
    SnapshotInfo snapshotInfo = getResponse.getSnapshots().get(0);
    assertThat(snapshotInfo.state(), equalTo(SnapshotState.IN_PROGRESS));
    logger.info("--> unblocking blocked node");
    unblockNode("test-repo", blockedNode);
    snapshotInfo = waitForCompletion("test-repo", "test-snap", TimeValue.timeValueSeconds(600));
    logger.info("Number of failed shards [{}]", snapshotInfo.shardFailures().size());
    logger.info("--> done");
    logger.info("--> checking snapshot status again after snapshot is done");
    response = client.admin().cluster().prepareSnapshotStatus("test-repo").addSnapshots("test-snap").execute().actionGet();
    snapshotStatus = response.getSnapshots().get(0);
    assertThat(snapshotStatus.getIndices().size(), equalTo(1));
    SnapshotIndexStatus indexStatus = snapshotStatus.getIndices().get("test-idx");
    assertThat(indexStatus, notNullValue());
    assertThat(indexStatus.getShardsStats().getInitializingShards(), equalTo(0));
    assertThat(indexStatus.getShardsStats().getFailedShards(), equalTo(snapshotInfo.failedShards()));
    assertThat(indexStatus.getShardsStats().getDoneShards(), equalTo(snapshotInfo.successfulShards()));
    assertThat(indexStatus.getShards().size(), equalTo(snapshotInfo.totalShards()));
    logger.info("--> checking snapshot status after it is done with empty repository");
    response = client.admin().cluster().prepareSnapshotStatus().execute().actionGet();
    assertThat(response.getSnapshots().size(), equalTo(0));
    logger.info("--> checking that _current no longer returns the snapshot");
    assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").addSnapshots("_current").execute().actionGet().getSnapshots().isEmpty(), equalTo(true));
    // test that getting an unavailable snapshot status throws an exception if ignoreUnavailable is false on the request
    SnapshotMissingException ex = expectThrows(SnapshotMissingException.class, () -> client.admin().cluster().prepareSnapshotStatus("test-repo").addSnapshots("test-snap-doesnt-exist").get());
    assertEquals("[test-repo:test-snap-doesnt-exist] is missing", ex.getMessage());
    // test that getting an unavailable snapshot status does not throw an exception if ignoreUnavailable is true on the request
    response = client.admin().cluster().prepareSnapshotStatus("test-repo").addSnapshots("test-snap-doesnt-exist").setIgnoreUnavailable(true).get();
    assertTrue(response.getSnapshots().isEmpty());
    // test getting snapshot status for available and unavailable snapshots where ignoreUnavailable is true
    // (available one should be returned)
    response = client.admin().cluster().prepareSnapshotStatus("test-repo").addSnapshots("test-snap", "test-snap-doesnt-exist").setIgnoreUnavailable(true).get();
    assertEquals(1, response.getSnapshots().size());
    assertEquals("test-snap", response.getSnapshots().get(0).getSnapshot().getSnapshotId().getName());
}

Example 25 with PutRepositoryResponse

use of org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryResponse in project elasticsearch by elastic.

the class SharedClusterSnapshotRestoreIT method testSnapshotCanceledOnRemovedShard.

/**
     * This test ensures that when a shard is removed from a node (perhaps due to the node
     * leaving the cluster, then returning), all snapshotting of that shard is aborted, so
     * all Store references held onto by the snapshot are released.
     *
     * See https://github.com/elastic/elasticsearch/issues/20876
     */
public void testSnapshotCanceledOnRemovedShard() throws Exception {
    final int numPrimaries = 1;
    final int numReplicas = 1;
    final int numDocs = 100;
    final String repo = "test-repo";
    final String index = "test-idx";
    final String snapshot = "test-snap";
    assertAcked(prepareCreate(index, 1, Settings.builder().put("number_of_shards", numPrimaries).put("number_of_replicas", numReplicas)));
    logger.info("--> indexing some data");
    for (int i = 0; i < numDocs; i++) {
        index(index, "doc", Integer.toString(i), "foo", "bar" + i);
    }
    refresh();
    logger.info("--> creating repository");
    PutRepositoryResponse putRepositoryResponse = client().admin().cluster().preparePutRepository(repo).setType("mock").setSettings(Settings.builder().put("location", randomRepoPath()).put("random", randomAsciiOfLength(10)).put("wait_after_unblock", 200)).get();
    assertTrue(putRepositoryResponse.isAcknowledged());
    String blockedNode = blockNodeWithIndex(repo, index);
    logger.info("--> snapshot");
    client().admin().cluster().prepareCreateSnapshot(repo, snapshot).setWaitForCompletion(false).execute();
    logger.info("--> waiting for block to kick in on node [{}]", blockedNode);
    waitForBlock(blockedNode, repo, TimeValue.timeValueSeconds(10));
    logger.info("--> removing primary shard that is being snapshotted");
    ClusterState clusterState = internalCluster().clusterService(internalCluster().getMasterName()).state();
    IndexRoutingTable indexRoutingTable = clusterState.getRoutingTable().index(index);
    String nodeWithPrimary = clusterState.nodes().get(indexRoutingTable.shard(0).primaryShard().currentNodeId()).getName();
    assertNotNull("should be at least one node with a primary shard", nodeWithPrimary);
    IndicesService indicesService = internalCluster().getInstance(IndicesService.class, nodeWithPrimary);
    IndexService indexService = indicesService.indexService(resolveIndex(index));
    indexService.removeShard(0, "simulate node removal");
    logger.info("--> unblocking blocked node [{}]", blockedNode);
    unblockNode(repo, blockedNode);
    logger.info("--> ensuring snapshot is aborted and the aborted shard was marked as failed");
    SnapshotInfo snapshotInfo = waitForCompletion(repo, snapshot, TimeValue.timeValueSeconds(10));
    assertEquals(1, snapshotInfo.shardFailures().size());
    assertEquals(0, snapshotInfo.shardFailures().get(0).shardId());
    assertEquals("IndexShardSnapshotFailedException[Aborted]", snapshotInfo.shardFailures().get(0).reason());
}