Examples with NetworkDisconnect org.elasticsearch.test.disruption.NetworkDisruption.NetworkDisconnect used on opensource projects

Example 6 with NetworkDisconnect

use of org.elasticsearch.test.disruption.NetworkDisruption.NetworkDisconnect in project elasticsearch by elastic.

the class DiscoveryWithServiceDisruptionsIT method testSendingShardFailure.

// simulate handling of sending shard failure during an isolation
public void testSendingShardFailure() throws Exception {
    List<String> nodes = startCluster(3, 2);
    String masterNode = internalCluster().getMasterName();
    List<String> nonMasterNodes = nodes.stream().filter(node -> !node.equals(masterNode)).collect(Collectors.toList());
    String nonMasterNode = randomFrom(nonMasterNodes);
    assertAcked(prepareCreate("test").setSettings(Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 3).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 2)));
    ensureGreen();
    String nonMasterNodeId = internalCluster().clusterService(nonMasterNode).localNode().getId();
    // fail a random shard
    ShardRouting failedShard = randomFrom(clusterService().state().getRoutingNodes().node(nonMasterNodeId).shardsWithState(ShardRoutingState.STARTED));
    ShardStateAction service = internalCluster().getInstance(ShardStateAction.class, nonMasterNode);
    CountDownLatch latch = new CountDownLatch(1);
    AtomicBoolean success = new AtomicBoolean();
    String isolatedNode = randomBoolean() ? masterNode : nonMasterNode;
    TwoPartitions partitions = isolateNode(isolatedNode);
    // we cannot use the NetworkUnresponsive disruption type here as it will swallow the "shard failed" request, calling neither
    // onSuccess nor onFailure on the provided listener.
    NetworkLinkDisruptionType disruptionType = new NetworkDisconnect();
    NetworkDisruption networkDisruption = new NetworkDisruption(partitions, disruptionType);
    setDisruptionScheme(networkDisruption);
    networkDisruption.startDisrupting();
    service.localShardFailed(failedShard, "simulated", new CorruptIndexException("simulated", (String) null), new ShardStateAction.Listener() {

        @Override
        public void onSuccess() {
            success.set(true);
            latch.countDown();
        }

        @Override
        public void onFailure(Exception e) {
            success.set(false);
            latch.countDown();
            assert false;
        }
    });
    if (isolatedNode.equals(nonMasterNode)) {
        assertNoMaster(nonMasterNode);
    } else {
        ensureStableCluster(2, nonMasterNode);
    }
    // heal the partition
    networkDisruption.removeAndEnsureHealthy(internalCluster());
    // the cluster should stabilize
    ensureStableCluster(3);
    latch.await();
    // the listener should be notified
    assertTrue(success.get());
    // the failed shard should be gone
    List<ShardRouting> shards = clusterService().state().getRoutingTable().allShards("test");
    for (ShardRouting shard : shards) {
        assertThat(shard.allocationId(), not(equalTo(failedShard.allocationId())));
    }
}

Example 7 with NetworkDisconnect

use of org.elasticsearch.test.disruption.NetworkDisruption.NetworkDisconnect in project elasticsearch by elastic.

the class DiscoveryWithServiceDisruptionsIT method addRandomDisruptionScheme.

private ServiceDisruptionScheme addRandomDisruptionScheme() {
    // TODO: add partial partitions
    final DisruptedLinks disruptedLinks;
    if (randomBoolean()) {
        disruptedLinks = TwoPartitions.random(random(), internalCluster().getNodeNames());
    } else {
        disruptedLinks = Bridge.random(random(), internalCluster().getNodeNames());
    }
    final NetworkLinkDisruptionType disruptionType;
    switch(randomInt(2)) {
        case 0:
            disruptionType = new NetworkUnresponsive();
            break;
        case 1:
            disruptionType = new NetworkDisconnect();
            break;
        case 2:
            disruptionType = NetworkDelay.random(random());
            break;
        default:
            throw new IllegalArgumentException();
    }
    final ServiceDisruptionScheme scheme;
    if (rarely()) {
        scheme = new SlowClusterStateProcessing(random());
    } else {
        scheme = new NetworkDisruption(disruptedLinks, disruptionType);
    }
    setDisruptionScheme(scheme);
    return scheme;
}

Example 8 with NetworkDisconnect

use of org.elasticsearch.test.disruption.NetworkDisruption.NetworkDisconnect in project elasticsearch by elastic.

the class DiscoveryWithServiceDisruptionsIT method testIsolatedUnicastNodes.

public void testIsolatedUnicastNodes() throws Exception {
    List<String> nodes = startCluster(4, -1, new int[] { 0 });
    // Figure out what is the elected master node
    final String unicastTarget = nodes.get(0);
    Set<String> unicastTargetSide = new HashSet<>();
    unicastTargetSide.add(unicastTarget);
    Set<String> restOfClusterSide = new HashSet<>();
    restOfClusterSide.addAll(nodes);
    restOfClusterSide.remove(unicastTarget);
    // Forcefully clean temporal response lists on all nodes. Otherwise the node in the unicast host list
    // includes all the other nodes that have pinged it and the issue doesn't manifest
    ZenPing zenPing = ((TestZenDiscovery) internalCluster().getInstance(Discovery.class)).getZenPing();
    if (zenPing instanceof UnicastZenPing) {
        ((UnicastZenPing) zenPing).clearTemporalResponses();
    }
    // Simulate a network issue between the unicast target node and the rest of the cluster
    NetworkDisruption networkDisconnect = new NetworkDisruption(new TwoPartitions(unicastTargetSide, restOfClusterSide), new NetworkDisconnect());
    setDisruptionScheme(networkDisconnect);
    networkDisconnect.startDisrupting();
    // Wait until elected master has removed that the unlucky node...
    ensureStableCluster(3, nodes.get(1));
    // The isolate master node must report no master, so it starts with pinging
    assertNoMaster(unicastTarget);
    networkDisconnect.stopDisrupting();
    // Wait until the master node sees all 3 nodes again.
    ensureStableCluster(4);
}

Example 9 with NetworkDisconnect

use of org.elasticsearch.test.disruption.NetworkDisruption.NetworkDisconnect in project elasticsearch by elastic.

the class DiscoveryWithServiceDisruptionsIT method addRandomDisruptionType.

protected NetworkDisruption addRandomDisruptionType(TwoPartitions partitions) {
    final NetworkLinkDisruptionType disruptionType;
    if (randomBoolean()) {
        disruptionType = new NetworkUnresponsive();
    } else {
        disruptionType = new NetworkDisconnect();
    }
    NetworkDisruption partition = new NetworkDisruption(partitions, disruptionType);
    setDisruptionScheme(partition);
    return partition;
}