Examples with MockTransportService org.elasticsearch.test.transport.MockTransportService used on opensource projects

Example 31 with MockTransportService

use of org.elasticsearch.test.transport.MockTransportService in project elasticsearch by elastic.

the class UnicastZenPingTests method startServices.

private NetworkHandle startServices(final Settings settings, final ThreadPool threadPool, final String nodeId, final Version version, final BiFunction<Settings, Version, Transport> supplier, final Set<Role> nodeRoles) {
    final Settings nodeSettings = Settings.builder().put(settings).put("node.name", nodeId).put(TransportService.TRACE_LOG_INCLUDE_SETTING.getKey(), "internal:discovery/zen/unicast").build();
    final Transport transport = supplier.apply(nodeSettings, version);
    final MockTransportService transportService = new MockTransportService(nodeSettings, transport, threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR, boundAddress -> new DiscoveryNode(nodeId, nodeId, boundAddress.publishAddress(), emptyMap(), nodeRoles, version), null);
    transportService.start();
    transportService.acceptIncomingRequests();
    final ConcurrentMap<TransportAddress, AtomicInteger> counters = ConcurrentCollections.newConcurrentMap();
    transportService.addTracer(new MockTransportService.Tracer() {

        @Override
        public void requestSent(DiscoveryNode node, long requestId, String action, TransportRequestOptions options) {
            counters.computeIfAbsent(node.getAddress(), k -> new AtomicInteger());
            counters.get(node.getAddress()).incrementAndGet();
        }
    });
    return new NetworkHandle(transport.boundAddress().publishAddress(), transportService, transportService.getLocalNode(), counters);
}

Example 32 with MockTransportService

use of org.elasticsearch.test.transport.MockTransportService in project elasticsearch by elastic.

the class PublishClusterStateActionTests method createMockNode.

public static MockNode createMockNode(String name, final Settings basSettings, @Nullable ClusterStateListener listener, ThreadPool threadPool, Logger logger, Map<String, MockNode> nodes) throws Exception {
    final Settings settings = Settings.builder().put("name", name).put(TransportService.TRACE_LOG_INCLUDE_SETTING.getKey(), "", TransportService.TRACE_LOG_EXCLUDE_SETTING.getKey(), "NOTHING").put(basSettings).build();
    MockTransportService service = buildTransportService(settings, threadPool);
    DiscoveryNode discoveryNode = service.getLocalDiscoNode();
    MockNode node = new MockNode(discoveryNode, service, listener, logger);
    node.action = buildPublishClusterStateAction(settings, service, () -> node.clusterState, node);
    final CountDownLatch latch = new CountDownLatch(nodes.size() * 2);
    TransportConnectionListener waitForConnection = new TransportConnectionListener() {

        @Override
        public void onNodeConnected(DiscoveryNode node) {
            latch.countDown();
        }

        @Override
        public void onNodeDisconnected(DiscoveryNode node) {
            fail("disconnect should not be called " + node);
        }
    };
    node.service.addConnectionListener(waitForConnection);
    for (MockNode curNode : nodes.values()) {
        curNode.service.addConnectionListener(waitForConnection);
        curNode.connectTo(node.discoveryNode);
        node.connectTo(curNode.discoveryNode);
    }
    assertThat("failed to wait for all nodes to connect", latch.await(5, TimeUnit.SECONDS), equalTo(true));
    for (MockNode curNode : nodes.values()) {
        curNode.service.removeConnectionListener(waitForConnection);
    }
    node.service.removeConnectionListener(waitForConnection);
    if (nodes.put(name, node) != null) {
        fail("Node with the name " + name + " already exist");
    }
    return node;
}

Example 33 with MockTransportService

use of org.elasticsearch.test.transport.MockTransportService in project elasticsearch by elastic.

the class ZenDiscoveryUnitTests method testNodesUpdatedAfterClusterStatePublished.

public void testNodesUpdatedAfterClusterStatePublished() throws Exception {
    ThreadPool threadPool = new TestThreadPool(getClass().getName());
    // randomly make minimum_master_nodes a value higher than we have nodes for, so it will force failure
    int minMasterNodes = randomBoolean() ? 3 : 1;
    Settings settings = Settings.builder().put(DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.toString(minMasterNodes)).build();
    ArrayDeque<Closeable> toClose = new ArrayDeque<>();
    try {
        Set<DiscoveryNode> expectedFDNodes = null;
        final MockTransportService masterTransport = MockTransportService.createNewService(settings, Version.CURRENT, threadPool, null);
        masterTransport.start();
        DiscoveryNode masterNode = masterTransport.getLocalNode();
        toClose.addFirst(masterTransport);
        ClusterState state = ClusterStateCreationUtils.state(masterNode, masterNode, masterNode);
        // build the zen discovery and cluster service
        ClusterService masterClusterService = createClusterService(threadPool, masterNode);
        toClose.addFirst(masterClusterService);
        // TODO: clustername shouldn't be stored twice in cluster service, but for now, work around it
        state = ClusterState.builder(masterClusterService.getClusterName()).nodes(state.nodes()).build();
        setState(masterClusterService, state);
        ZenDiscovery masterZen = buildZenDiscovery(settings, masterTransport, masterClusterService, threadPool);
        toClose.addFirst(masterZen);
        masterTransport.acceptIncomingRequests();
        final MockTransportService otherTransport = MockTransportService.createNewService(settings, Version.CURRENT, threadPool, null);
        otherTransport.start();
        toClose.addFirst(otherTransport);
        DiscoveryNode otherNode = otherTransport.getLocalNode();
        final ClusterState otherState = ClusterState.builder(masterClusterService.getClusterName()).nodes(DiscoveryNodes.builder().add(otherNode).localNodeId(otherNode.getId())).build();
        ClusterService otherClusterService = createClusterService(threadPool, masterNode);
        toClose.addFirst(otherClusterService);
        setState(otherClusterService, otherState);
        ZenDiscovery otherZen = buildZenDiscovery(settings, otherTransport, otherClusterService, threadPool);
        toClose.addFirst(otherZen);
        otherTransport.acceptIncomingRequests();
        masterTransport.connectToNode(otherNode);
        otherTransport.connectToNode(masterNode);
        // a new cluster state with a new discovery node (we will test if the cluster state
        // was updated by the presence of this node in NodesFaultDetection)
        ClusterState newState = ClusterState.builder(masterClusterService.state()).incrementVersion().nodes(DiscoveryNodes.builder(state.nodes()).add(otherNode).masterNodeId(masterNode.getId())).build();
        try {
            // publishing a new cluster state
            ClusterChangedEvent clusterChangedEvent = new ClusterChangedEvent("testing", newState, state);
            AssertingAckListener listener = new AssertingAckListener(newState.nodes().getSize() - 1);
            expectedFDNodes = masterZen.getFaultDetectionNodes();
            masterZen.publish(clusterChangedEvent, listener);
            listener.await(1, TimeUnit.HOURS);
            // publish was a success, update expected FD nodes based on new cluster state
            expectedFDNodes = fdNodesForState(newState, masterNode);
        } catch (Discovery.FailedToCommitClusterStateException e) {
            // not successful, so expectedFDNodes above should remain what it was originally assigned
            // ensure min master nodes is the higher value, otherwise we shouldn't fail
            assertEquals(3, minMasterNodes);
        }
        assertEquals(expectedFDNodes, masterZen.getFaultDetectionNodes());
    } finally {
        IOUtils.close(toClose);
        terminate(threadPool);
    }
}

Example 34 with MockTransportService

use of org.elasticsearch.test.transport.MockTransportService in project elasticsearch by elastic.

the class CorruptedFileIT method testCorruptionOnNetworkLayerFinalizingRecovery.

/**
     * This test triggers a corrupt index exception during finalization size if an empty commit point is transferred
     * during recovery we don't know the version of the segments_N file because it has no segments we can take it from.
     * This simulates recoveries from old indices or even without checksums and makes sure if we fail during finalization
     * we also check if the primary is ok. Without the relevant checks this test fails with a RED cluster
     */
public void testCorruptionOnNetworkLayerFinalizingRecovery() throws ExecutionException, InterruptedException, IOException {
    internalCluster().ensureAtLeastNumDataNodes(2);
    NodesStatsResponse nodeStats = client().admin().cluster().prepareNodesStats().get();
    List<NodeStats> dataNodeStats = new ArrayList<>();
    for (NodeStats stat : nodeStats.getNodes()) {
        if (stat.getNode().isDataNode()) {
            dataNodeStats.add(stat);
        }
    }
    assertThat(dataNodeStats.size(), greaterThanOrEqualTo(2));
    Collections.shuffle(dataNodeStats, random());
    NodeStats primariesNode = dataNodeStats.get(0);
    NodeStats unluckyNode = dataNodeStats.get(1);
    assertAcked(prepareCreate("test").setSettings(Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "0").put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).put("index.routing.allocation.include._name", primariesNode.getNode().getName()).put(EnableAllocationDecider.INDEX_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), EnableAllocationDecider.Rebalance.NONE).put("index.allocation.max_retries", // keep on retrying
    Integer.MAX_VALUE)));
    // allocated with empty commit
    ensureGreen();
    final AtomicBoolean corrupt = new AtomicBoolean(true);
    final CountDownLatch hasCorrupted = new CountDownLatch(1);
    for (NodeStats dataNode : dataNodeStats) {
        MockTransportService mockTransportService = ((MockTransportService) internalCluster().getInstance(TransportService.class, dataNode.getNode().getName()));
        mockTransportService.addDelegate(internalCluster().getInstance(TransportService.class, unluckyNode.getNode().getName()), new MockTransportService.DelegateTransport(mockTransportService.original()) {

            @Override
            protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request, TransportRequestOptions options) throws IOException {
                if (corrupt.get() && action.equals(PeerRecoveryTargetService.Actions.FILE_CHUNK)) {
                    RecoveryFileChunkRequest req = (RecoveryFileChunkRequest) request;
                    byte[] array = BytesRef.deepCopyOf(req.content().toBytesRef()).bytes;
                    int i = randomIntBetween(0, req.content().length() - 1);
                    // flip one byte in the content
                    array[i] = (byte) ~array[i];
                    hasCorrupted.countDown();
                }
                super.sendRequest(connection, requestId, action, request, options);
            }
        });
    }
    Settings build = Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "1").put("index.routing.allocation.include._name", primariesNode.getNode().getName() + "," + unluckyNode.getNode().getName()).build();
    client().admin().indices().prepareUpdateSettings("test").setSettings(build).get();
    client().admin().cluster().prepareReroute().get();
    hasCorrupted.await();
    corrupt.set(false);
    ensureGreen();
}

Example 35 with MockTransportService

use of org.elasticsearch.test.transport.MockTransportService in project elasticsearch by elastic.

the class CorruptedFileIT method testCorruptionOnNetworkLayer.

/**
     * Tests corruption that happens on the network layer and that the primary does not get affected by corruption that happens on the way
     * to the replica. The file on disk stays uncorrupted
     */
public void testCorruptionOnNetworkLayer() throws ExecutionException, InterruptedException {
    int numDocs = scaledRandomIntBetween(100, 1000);
    internalCluster().ensureAtLeastNumDataNodes(2);
    if (cluster().numDataNodes() < 3) {
        internalCluster().startNode(Settings.builder().put(Node.NODE_DATA_SETTING.getKey(), true).put(Node.NODE_MASTER_SETTING.getKey(), false));
    }
    NodesStatsResponse nodeStats = client().admin().cluster().prepareNodesStats().get();
    List<NodeStats> dataNodeStats = new ArrayList<>();
    for (NodeStats stat : nodeStats.getNodes()) {
        if (stat.getNode().isDataNode()) {
            dataNodeStats.add(stat);
        }
    }
    assertThat(dataNodeStats.size(), greaterThanOrEqualTo(2));
    Collections.shuffle(dataNodeStats, random());
    NodeStats primariesNode = dataNodeStats.get(0);
    NodeStats unluckyNode = dataNodeStats.get(1);
    assertAcked(prepareCreate("test").setSettings(Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "0").put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, // don't go crazy here it must recovery fast
    between(1, 4)).put(MockFSIndexStore.INDEX_CHECK_INDEX_ON_CLOSE_SETTING.getKey(), false).put("index.routing.allocation.include._name", primariesNode.getNode().getName()).put(EnableAllocationDecider.INDEX_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), EnableAllocationDecider.Rebalance.NONE)));
    ensureGreen();
    IndexRequestBuilder[] builders = new IndexRequestBuilder[numDocs];
    for (int i = 0; i < builders.length; i++) {
        builders[i] = client().prepareIndex("test", "type").setSource("field", "value");
    }
    indexRandom(true, builders);
    ensureGreen();
    assertAllSuccessful(client().admin().indices().prepareFlush().setForce(true).execute().actionGet());
    // we have to flush at least once here since we don't corrupt the translog
    SearchResponse countResponse = client().prepareSearch().setSize(0).get();
    assertHitCount(countResponse, numDocs);
    final boolean truncate = randomBoolean();
    for (NodeStats dataNode : dataNodeStats) {
        MockTransportService mockTransportService = ((MockTransportService) internalCluster().getInstance(TransportService.class, dataNode.getNode().getName()));
        mockTransportService.addDelegate(internalCluster().getInstance(TransportService.class, unluckyNode.getNode().getName()), new MockTransportService.DelegateTransport(mockTransportService.original()) {

            @Override
            protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request, TransportRequestOptions options) throws IOException {
                if (action.equals(PeerRecoveryTargetService.Actions.FILE_CHUNK)) {
                    RecoveryFileChunkRequest req = (RecoveryFileChunkRequest) request;
                    if (truncate && req.length() > 1) {
                        BytesRef bytesRef = req.content().toBytesRef();
                        BytesArray array = new BytesArray(bytesRef.bytes, bytesRef.offset, (int) req.length() - 1);
                        request = new RecoveryFileChunkRequest(req.recoveryId(), req.shardId(), req.metadata(), req.position(), array, req.lastChunk(), req.totalTranslogOps(), req.sourceThrottleTimeInNanos());
                    } else {
                        assert req.content().toBytesRef().bytes == req.content().toBytesRef().bytes : "no internal reference!!";
                        final byte[] array = req.content().toBytesRef().bytes;
                        int i = randomIntBetween(0, req.content().length() - 1);
                        // flip one byte in the content
                        array[i] = (byte) ~array[i];
                    }
                }
                super.sendRequest(connection, requestId, action, request, options);
            }
        });
    }
    Settings build = Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, "1").put("index.routing.allocation.include._name", "*").build();
    client().admin().indices().prepareUpdateSettings("test").setSettings(build).get();
    client().admin().cluster().prepareReroute().get();
    ClusterHealthResponse actionGet = client().admin().cluster().health(Requests.clusterHealthRequest("test").waitForGreenStatus()).actionGet();
    if (actionGet.isTimedOut()) {
        logger.info("ensureGreen timed out, cluster state:\n{}\n{}", client().admin().cluster().prepareState().get().getState(), client().admin().cluster().preparePendingClusterTasks().get());
        assertThat("timed out waiting for green state", actionGet.isTimedOut(), equalTo(false));
    }
    // we are green so primaries got not corrupted.
    // ensure that no shard is actually allocated on the unlucky node
    ClusterStateResponse clusterStateResponse = client().admin().cluster().prepareState().get();
    for (IndexShardRoutingTable table : clusterStateResponse.getState().getRoutingTable().index("test")) {
        for (ShardRouting routing : table) {
            if (unluckyNode.getNode().getId().equals(routing.currentNodeId())) {
                assertThat(routing.state(), not(equalTo(ShardRoutingState.STARTED)));
                assertThat(routing.state(), not(equalTo(ShardRoutingState.RELOCATING)));
            }
        }
    }
    final int numIterations = scaledRandomIntBetween(5, 20);
    for (int i = 0; i < numIterations; i++) {
        SearchResponse response = client().prepareSearch().setSize(numDocs).get();
        assertHitCount(response, numDocs);
    }
}