Examples with MockTransportService org.elasticsearch.test.transport.MockTransportService used on opensource projects

Example 51 with MockTransportService

use of org.elasticsearch.test.transport.MockTransportService in project crate by crate.

the class IndexRecoveryIT method testCancelNewShardRecoveryAndUsesExistingShardCopy.

@Test
public void testCancelNewShardRecoveryAndUsesExistingShardCopy() throws Exception {
    logger.info("--> start node A");
    final String nodeA = internalCluster().startNode();
    logger.info("--> create index on node: {}", nodeA);
    createAndPopulateIndex(INDEX_NAME, 1, SHARD_COUNT, REPLICA_COUNT);
    logger.info("--> start node B");
    // force a shard recovery from nodeA to nodeB
    final String nodeB = internalCluster().startNode();
    logger.info("--> add replica for {} on node: {}", INDEX_NAME, nodeB);
    execute("ALTER TABLE " + INDEX_NAME + " SET (number_of_replicas=1, \"unassigned.node_left.delayed_timeout\"=0)");
    ensureGreen();
    logger.info("--> start node C");
    final String nodeC = internalCluster().startNode();
    // do sync flush to gen sync id
    execute("OPTIMIZE TABLE " + INDEX_NAME);
    // assertThat(client().admin().indices().prepareSyncedFlush(INDEX_NAME).get().failedShards(), equalTo(0));
    // hold peer recovery on phase 2 after nodeB down
    CountDownLatch phase1ReadyBlocked = new CountDownLatch(1);
    CountDownLatch allowToCompletePhase1Latch = new CountDownLatch(1);
    MockTransportService transportService = (MockTransportService) internalCluster().getInstance(TransportService.class, nodeA);
    transportService.addSendBehavior((connection, requestId, action, request, options) -> {
        if (PeerRecoveryTargetService.Actions.CLEAN_FILES.equals(action)) {
            phase1ReadyBlocked.countDown();
            try {
                allowToCompletePhase1Latch.await();
            } catch (InterruptedException e) {
                throw new AssertionError(e);
            }
        }
        connection.sendRequest(requestId, action, request, options);
    });
    logger.info("--> restart node B");
    internalCluster().restartNode(nodeB, new InternalTestCluster.RestartCallback() {

        @Override
        public Settings onNodeStopped(String nodeName) throws Exception {
            phase1ReadyBlocked.await();
            // nodeB stopped, peer recovery from nodeA to nodeC, it will be cancelled after nodeB get started.
            var indexName = IndexParts.toIndexName(sqlExecutor.getCurrentSchema(), INDEX_NAME, null);
            RecoveryResponse response = client().execute(RecoveryAction.INSTANCE, new RecoveryRequest(indexName)).actionGet();
            List<RecoveryState> recoveryStates = response.shardRecoveryStates().get(indexName);
            List<RecoveryState> nodeCRecoveryStates = findRecoveriesForTargetNode(nodeC, recoveryStates);
            assertThat(nodeCRecoveryStates.size(), equalTo(1));
            assertOnGoingRecoveryState(nodeCRecoveryStates.get(0), 0, RecoverySource.PeerRecoverySource.INSTANCE, false, nodeA, nodeC);
            validateIndexRecoveryState(nodeCRecoveryStates.get(0).getIndex());
            return super.onNodeStopped(nodeName);
        }
    });
    // wait for peer recovery from nodeA to nodeB which is a no-op recovery so it skips the CLEAN_FILES stage and hence is not blocked
    ensureGreen();
    allowToCompletePhase1Latch.countDown();
    transportService.clearAllRules();
    // make sure nodeA has primary and nodeB has replica
    ClusterState state = client().admin().cluster().prepareState().get().getState();
    List<ShardRouting> startedShards = state.routingTable().shardsWithState(ShardRoutingState.STARTED);
    assertThat(startedShards.size(), equalTo(2));
    for (ShardRouting shardRouting : startedShards) {
        if (shardRouting.primary()) {
            assertThat(state.nodes().get(shardRouting.currentNodeId()).getName(), equalTo(nodeA));
        } else {
            assertThat(state.nodes().get(shardRouting.currentNodeId()).getName(), equalTo(nodeB));
        }
    }
}

Example 52 with MockTransportService

use of org.elasticsearch.test.transport.MockTransportService in project crate by crate.

the class IndexRecoveryIT method testDoNotInfinitelyWaitForMapping.

@Test
public void testDoNotInfinitelyWaitForMapping() {
    internalCluster().ensureAtLeastNumDataNodes(3);
    execute("CREATE ANALYZER test_analyzer (" + " TOKENIZER standard," + " TOKEN_FILTERS (test_token_filter)" + ")");
    execute("CREATE TABLE test (test_field TEXT INDEX USING FULLTEXT WITH (ANALYZER='test_analyzer'))" + " CLUSTERED INTO 1 SHARDS WITH (number_of_replicas = 0)");
    int numDocs = between(1, 10);
    var args = new Object[numDocs][];
    for (int i = 0; i < numDocs; i++) {
        args[i] = new Object[] { Integer.toString(i) };
    }
    execute("INSERT INTO test (test_field) VALUES (?)", args);
    Semaphore recoveryBlocked = new Semaphore(1);
    for (DiscoveryNode node : clusterService().state().nodes()) {
        MockTransportService transportService = (MockTransportService) internalCluster().getInstance(TransportService.class, node.getName());
        transportService.addSendBehavior((connection, requestId, action, request, options) -> {
            if (action.equals(PeerRecoverySourceService.Actions.START_RECOVERY)) {
                if (recoveryBlocked.tryAcquire()) {
                    PluginsService pluginService = internalCluster().getInstance(PluginsService.class, node.getName());
                    for (TestAnalysisPlugin plugin : pluginService.filterPlugins(TestAnalysisPlugin.class)) {
                        plugin.throwParsingError.set(true);
                    }
                }
            }
            connection.sendRequest(requestId, action, request, options);
        });
    }
    execute("ALTER TABLE test SET (number_of_replicas = 1)");
    ensureGreen();
    refresh();
    var searchResponse = execute("SELECT COUNT(*) FROM test");
    assertThat((long) searchResponse.rows()[0][0], is((long) numDocs));
}

Example 53 with MockTransportService

use of org.elasticsearch.test.transport.MockTransportService in project crate by crate.

the class IndexRecoveryIT method testRecoverLocallyUpToGlobalCheckpoint.

@Test
public void testRecoverLocallyUpToGlobalCheckpoint() throws Exception {
    internalCluster().ensureAtLeastNumDataNodes(2);
    List<String> nodes = randomSubsetOf(2, StreamSupport.stream(clusterService().state().nodes().getDataNodes().spliterator(), false).map(node -> node.value.getName()).collect(Collectors.toSet()));
    String indexName = "test";
    execute("CREATE TABLE doc.test (num INT)" + " CLUSTERED INTO 1 SHARDS" + " WITH (" + "  number_of_replicas = 1," + "  \"global_checkpoint_sync.interval\"='24h'," + "  \"routing.allocation.include._name\"='" + String.join(",", nodes) + "'" + " )");
    ensureGreen(indexName);
    int numDocs = randomIntBetween(1, 100);
    var args = new Object[numDocs][];
    for (int i = 0; i < numDocs; i++) {
        args[i] = new Object[] { i };
    }
    execute("INSERT INTO doc.test (num) VALUES (?)", args);
    refresh(indexName);
    String failingNode = randomFrom(nodes);
    PlainActionFuture<StartRecoveryRequest> startRecoveryRequestFuture = new PlainActionFuture<>();
    // Peer recovery fails if the primary does not see the recovering replica in the replication group (when the cluster state
    // update on the primary is delayed). To verify the local recovery stats, we have to manually remember this value in the
    // first try because the local recovery happens once and its stats is reset when the recovery fails.
    SetOnce<Integer> localRecoveredOps = new SetOnce<>();
    for (String node : nodes) {
        MockTransportService transportService = (MockTransportService) internalCluster().getInstance(TransportService.class, node);
        transportService.addSendBehavior((connection, requestId, action, request, options) -> {
            if (action.equals(PeerRecoverySourceService.Actions.START_RECOVERY)) {
                final RecoveryState recoveryState = internalCluster().getInstance(IndicesService.class, failingNode).getShardOrNull(new ShardId(resolveIndex(indexName), 0)).recoveryState();
                assertThat(recoveryState.getTranslog().recoveredOperations(), equalTo(recoveryState.getTranslog().totalLocal()));
                if (startRecoveryRequestFuture.isDone()) {
                    assertThat(recoveryState.getTranslog().totalLocal(), equalTo(0));
                    recoveryState.getTranslog().totalLocal(localRecoveredOps.get());
                    recoveryState.getTranslog().incrementRecoveredOperations(localRecoveredOps.get());
                } else {
                    localRecoveredOps.set(recoveryState.getTranslog().totalLocal());
                    startRecoveryRequestFuture.onResponse((StartRecoveryRequest) request);
                }
            }
            if (action.equals(PeerRecoveryTargetService.Actions.FILE_CHUNK)) {
                RetentionLeases retentionLeases = internalCluster().getInstance(IndicesService.class, node).indexServiceSafe(resolveIndex(indexName)).getShard(0).getRetentionLeases();
                throw new AssertionError("expect an operation-based recovery:" + "retention leases" + Strings.toString(retentionLeases) + "]");
            }
            connection.sendRequest(requestId, action, request, options);
        });
    }
    IndexShard shard = internalCluster().getInstance(IndicesService.class, failingNode).getShardOrNull(new ShardId(resolveIndex(indexName), 0));
    final long lastSyncedGlobalCheckpoint = shard.getLastSyncedGlobalCheckpoint();
    final long localCheckpointOfSafeCommit;
    try (Engine.IndexCommitRef safeCommitRef = shard.acquireSafeIndexCommit()) {
        localCheckpointOfSafeCommit = SequenceNumbers.loadSeqNoInfoFromLuceneCommit(safeCommitRef.getIndexCommit().getUserData().entrySet()).localCheckpoint;
    }
    final long maxSeqNo = shard.seqNoStats().getMaxSeqNo();
    shard.failShard("test", new IOException("simulated"));
    StartRecoveryRequest startRecoveryRequest = startRecoveryRequestFuture.actionGet();
    logger.info("--> start recovery request: starting seq_no {}, commit {}", startRecoveryRequest.startingSeqNo(), startRecoveryRequest.metadataSnapshot().getCommitUserData());
    SequenceNumbers.CommitInfo commitInfoAfterLocalRecovery = SequenceNumbers.loadSeqNoInfoFromLuceneCommit(startRecoveryRequest.metadataSnapshot().getCommitUserData().entrySet());
    assertThat(commitInfoAfterLocalRecovery.localCheckpoint, equalTo(lastSyncedGlobalCheckpoint));
    assertThat(commitInfoAfterLocalRecovery.maxSeqNo, equalTo(lastSyncedGlobalCheckpoint));
    assertThat(startRecoveryRequest.startingSeqNo(), equalTo(lastSyncedGlobalCheckpoint + 1));
    ensureGreen(indexName);
    assertThat((long) localRecoveredOps.get(), equalTo(lastSyncedGlobalCheckpoint - localCheckpointOfSafeCommit));
    for (var recoveryState : client().execute(RecoveryAction.INSTANCE, new RecoveryRequest()).get().shardRecoveryStates().get(indexName)) {
        if (startRecoveryRequest.targetNode().equals(recoveryState.getTargetNode())) {
            assertThat("expect an operation-based recovery", recoveryState.getIndex().fileDetails().values(), empty());
            assertThat("total recovered translog operations must include both local and remote recovery", recoveryState.getTranslog().recoveredOperations(), greaterThanOrEqualTo(Math.toIntExact(maxSeqNo - localCheckpointOfSafeCommit)));
        }
    }
    for (String node : nodes) {
        MockTransportService transportService = (MockTransportService) internalCluster().getInstance(TransportService.class, node);
        transportService.clearAllRules();
    }
}

Example 54 with MockTransportService

use of org.elasticsearch.test.transport.MockTransportService in project crate by crate.

the class IndexRecoveryIT method testDisconnectsWhileRecovering.

@Test
public void testDisconnectsWhileRecovering() throws Exception {
    final String indexName = "test";
    final Settings nodeSettings = Settings.builder().put(RecoverySettings.INDICES_RECOVERY_RETRY_DELAY_NETWORK_SETTING.getKey(), "100ms").put(RecoverySettings.INDICES_RECOVERY_INTERNAL_ACTION_TIMEOUT_SETTING.getKey(), "1s").put(NodeConnectionsService.CLUSTER_NODE_RECONNECT_INTERVAL_SETTING.getKey(), "1s").build();
    // start a master node
    internalCluster().startNode(nodeSettings);
    final String blueNodeName = internalCluster().startNode(Settings.builder().put("node.attr.color", "blue").put(nodeSettings).build());
    final String redNodeName = internalCluster().startNode(Settings.builder().put("node.attr.color", "red").put(nodeSettings).build());
    ClusterHealthResponse response = client().admin().cluster().prepareHealth().setWaitForNodes(">=3").get();
    assertThat(response.isTimedOut(), is(false));
    execute("CREATE TABLE doc." + indexName + " (id int) CLUSTERED INTO 1 SHARDS " + "WITH (" + " number_of_replicas=0," + " \"routing.allocation.include.color\" = 'blue'" + ")");
    int numDocs = scaledRandomIntBetween(25, 250);
    var args = new Object[numDocs][];
    for (int i = 0; i < numDocs; i++) {
        args[i] = new Object[] { i };
    }
    execute("INSERT INTO doc." + indexName + " (id) VALUES (?)", args);
    ensureGreen();
    ClusterStateResponse stateResponse = client().admin().cluster().prepareState().get();
    final String blueNodeId = internalCluster().getInstance(ClusterService.class, blueNodeName).localNode().getId();
    assertFalse(stateResponse.getState().getRoutingNodes().node(blueNodeId).isEmpty());
    refresh();
    var searchResponse = execute("SELECT COUNT(*) FROM doc." + indexName);
    assertThat((long) searchResponse.rows()[0][0], is((long) numDocs));
    String[] recoveryActions = new String[] { PeerRecoverySourceService.Actions.START_RECOVERY, PeerRecoveryTargetService.Actions.FILES_INFO, PeerRecoveryTargetService.Actions.FILE_CHUNK, PeerRecoveryTargetService.Actions.CLEAN_FILES, // RecoveryTarget.Actions.TRANSLOG_OPS, <-- may not be sent if already flushed
    PeerRecoveryTargetService.Actions.PREPARE_TRANSLOG, PeerRecoveryTargetService.Actions.FINALIZE };
    final String recoveryActionToBlock = randomFrom(recoveryActions);
    final boolean dropRequests = randomBoolean();
    logger.info("--> will {} between blue & red on [{}]", dropRequests ? "drop requests" : "break connection", recoveryActionToBlock);
    MockTransportService blueMockTransportService = (MockTransportService) internalCluster().getInstance(TransportService.class, blueNodeName);
    MockTransportService redMockTransportService = (MockTransportService) internalCluster().getInstance(TransportService.class, redNodeName);
    TransportService redTransportService = internalCluster().getInstance(TransportService.class, redNodeName);
    TransportService blueTransportService = internalCluster().getInstance(TransportService.class, blueNodeName);
    final CountDownLatch requestFailed = new CountDownLatch(1);
    if (randomBoolean()) {
        // Fail on the sending side
        blueMockTransportService.addSendBehavior(redTransportService, new RecoveryActionBlocker(dropRequests, recoveryActionToBlock, requestFailed));
        redMockTransportService.addSendBehavior(blueTransportService, new RecoveryActionBlocker(dropRequests, recoveryActionToBlock, requestFailed));
    } else {
        // Fail on the receiving side.
        blueMockTransportService.addRequestHandlingBehavior(recoveryActionToBlock, (handler, request, channel) -> {
            logger.info("--> preventing {} response by closing response channel", recoveryActionToBlock);
            requestFailed.countDown();
            redMockTransportService.disconnectFromNode(blueMockTransportService.getLocalNode());
            handler.messageReceived(request, channel);
        });
        redMockTransportService.addRequestHandlingBehavior(recoveryActionToBlock, (handler, request, channel) -> {
            logger.info("--> preventing {} response by closing response channel", recoveryActionToBlock);
            requestFailed.countDown();
            blueMockTransportService.disconnectFromNode(redMockTransportService.getLocalNode());
            handler.messageReceived(request, channel);
        });
    }
    logger.info("--> starting recovery from blue to red");
    execute("ALTER TABLE doc." + indexName + " SET (" + " number_of_replicas=1," + " \"routing.allocation.include.color\" = 'red,blue'" + ")");
    requestFailed.await();
    logger.info("--> clearing rules to allow recovery to proceed");
    blueMockTransportService.clearAllRules();
    redMockTransportService.clearAllRules();
    ensureGreen();
    // TODO: ES will use the shard routing preference here to prefer `_local` shards on that node
    var nodeRedExecutor = executor(redNodeName);
    searchResponse = nodeRedExecutor.exec("SELECT COUNT(*) FROM doc." + indexName);
    assertThat((long) searchResponse.rows()[0][0], is((long) numDocs));
}

Example 55 with MockTransportService

use of org.elasticsearch.test.transport.MockTransportService in project crate by crate.

the class IndexRecoveryIT method testDisconnectsDuringRecovery.

/**
 * Tests scenario where recovery target successfully sends recovery request to source but then the channel gets closed while
 * the source is working on the recovery process.
 */
@Test
public void testDisconnectsDuringRecovery() throws Exception {
    boolean primaryRelocation = randomBoolean();
    final String indexName = IndexParts.toIndexName(sqlExecutor.getCurrentSchema(), "test", null);
    final Settings nodeSettings = Settings.builder().put(RecoverySettings.INDICES_RECOVERY_RETRY_DELAY_NETWORK_SETTING.getKey(), TimeValue.timeValueMillis(randomIntBetween(0, 100))).build();
    TimeValue disconnectAfterDelay = TimeValue.timeValueMillis(randomIntBetween(0, 100));
    // start a master node
    String masterNodeName = internalCluster().startMasterOnlyNode(nodeSettings);
    final String blueNodeName = internalCluster().startNode(Settings.builder().put("node.attr.color", "blue").put(nodeSettings).build());
    final String redNodeName = internalCluster().startNode(Settings.builder().put("node.attr.color", "red").put(nodeSettings).build());
    execute("CREATE TABLE test (id int) CLUSTERED INTO 1 SHARDS " + "WITH (" + " number_of_replicas=0," + " \"routing.allocation.include.color\" = 'blue'" + ")");
    int numDocs = scaledRandomIntBetween(25, 250);
    var args = new Object[numDocs][];
    for (int i = 0; i < numDocs; i++) {
        args[i] = new Object[] { i };
    }
    execute("INSERT INTO test (id) VALUES (?)", args);
    ensureGreen();
    refresh();
    var searchResponse = execute("SELECT COUNT(*) FROM test");
    assertThat((long) searchResponse.rows()[0][0], is((long) numDocs));
    MockTransportService masterTransportService = (MockTransportService) internalCluster().getInstance(TransportService.class, masterNodeName);
    MockTransportService blueMockTransportService = (MockTransportService) internalCluster().getInstance(TransportService.class, blueNodeName);
    MockTransportService redMockTransportService = (MockTransportService) internalCluster().getInstance(TransportService.class, redNodeName);
    redMockTransportService.addSendBehavior(blueMockTransportService, new StubbableTransport.SendRequestBehavior() {

        private final AtomicInteger count = new AtomicInteger();

        @Override
        public void sendRequest(Transport.Connection connection, long requestId, String action, TransportRequest request, TransportRequestOptions options) throws IOException {
            logger.info("--> sending request {} on {}", action, connection.getNode());
            if (PeerRecoverySourceService.Actions.START_RECOVERY.equals(action) && count.incrementAndGet() == 1) {
                // ensures that it's considered as valid recovery attempt by source
                try {
                    assertBusy(() -> assertThat("Expected there to be some initializing shards", client(blueNodeName).admin().cluster().prepareState().setLocal(true).get().getState().getRoutingTable().index(indexName).shard(0).getAllInitializingShards(), not(empty())));
                } catch (Exception e) {
                    throw new RuntimeException(e);
                }
                connection.sendRequest(requestId, action, request, options);
                try {
                    Thread.sleep(disconnectAfterDelay.millis());
                } catch (InterruptedException e) {
                    throw new RuntimeException(e);
                }
                throw new ConnectTransportException(connection.getNode(), "DISCONNECT: simulation disconnect after successfully sending " + action + " request");
            } else {
                connection.sendRequest(requestId, action, request, options);
            }
        }
    });
    final AtomicBoolean finalized = new AtomicBoolean();
    blueMockTransportService.addSendBehavior(redMockTransportService, (connection, requestId, action, request, options) -> {
        logger.info("--> sending request {} on {}", action, connection.getNode());
        if (action.equals(PeerRecoveryTargetService.Actions.FINALIZE)) {
            finalized.set(true);
        }
        connection.sendRequest(requestId, action, request, options);
    });
    for (MockTransportService mockTransportService : Arrays.asList(redMockTransportService, blueMockTransportService)) {
        mockTransportService.addSendBehavior(masterTransportService, (connection, requestId, action, request, options) -> {
            logger.info("--> sending request {} on {}", action, connection.getNode());
            if ((primaryRelocation && finalized.get()) == false) {
                assertNotEquals(action, ShardStateAction.SHARD_FAILED_ACTION_NAME);
            }
            connection.sendRequest(requestId, action, request, options);
        });
    }
    if (primaryRelocation) {
        logger.info("--> starting primary relocation recovery from blue to red");
        execute("ALTER TABLE test SET (" + " \"routing.allocation.include.color\" = 'red'" + ")");
        // also waits for relocation / recovery to complete
        ensureGreen();
        // if a primary relocation fails after the source shard has been marked as relocated, both source and target are failed. If the
        // source shard is moved back to started because the target fails first, it's possible that there is a cluster state where the
        // shard is marked as started again (and ensureGreen returns), but while applying the cluster state the primary is failed and
        // will be reallocated. The cluster will thus become green, then red, then green again. Triggering a refresh here before
        // searching helps, as in contrast to search actions, refresh waits for the closed shard to be reallocated.
        refresh();
    } else {
        logger.info("--> starting replica recovery from blue to red");
        execute("ALTER TABLE test SET (" + " number_of_replicas=1," + " \"routing.allocation.include.color\" = 'red,blue'" + ")");
        ensureGreen();
    }
    for (int i = 0; i < 10; i++) {
        searchResponse = execute("SELECT COUNT(*) FROM test");
        assertThat((long) searchResponse.rows()[0][0], is((long) numDocs));
    }
}