Examples with DeterministicTaskQueue org.opensearch.cluster.coordination.DeterministicTaskQueue used on opensource projects

Example 1 with DeterministicTaskQueue

use of org.opensearch.cluster.coordination.DeterministicTaskQueue in project OpenSearch by opensearch-project.

the class NodeConnectionsServiceTests method testPeriodicReconnection.

public void testPeriodicReconnection() {
    final Settings.Builder settings = Settings.builder();
    final long reconnectIntervalMillis;
    if (randomBoolean()) {
        reconnectIntervalMillis = CLUSTER_NODE_RECONNECT_INTERVAL_SETTING.get(Settings.EMPTY).millis();
    } else {
        reconnectIntervalMillis = randomLongBetween(1, 100000);
        settings.put(CLUSTER_NODE_RECONNECT_INTERVAL_SETTING.getKey(), reconnectIntervalMillis + "ms");
    }
    final DeterministicTaskQueue deterministicTaskQueue = new DeterministicTaskQueue(builder().put(NODE_NAME_SETTING.getKey(), "node").build(), random());
    MockTransport transport = new MockTransport(deterministicTaskQueue.getThreadPool());
    TestTransportService transportService = new TestTransportService(transport, deterministicTaskQueue.getThreadPool());
    transportService.start();
    transportService.acceptIncomingRequests();
    final NodeConnectionsService service = new NodeConnectionsService(settings.build(), deterministicTaskQueue.getThreadPool(), transportService);
    service.start();
    final List<DiscoveryNode> allNodes = generateNodes();
    final DiscoveryNodes targetNodes = discoveryNodesFromList(randomSubsetOf(allNodes));
    transport.randomConnectionExceptions = true;
    final AtomicBoolean connectionCompleted = new AtomicBoolean();
    service.connectToNodes(targetNodes, () -> connectionCompleted.set(true));
    deterministicTaskQueue.runAllRunnableTasks();
    assertTrue(connectionCompleted.get());
    long maxDisconnectionTime = 0;
    for (int iteration = 0; iteration < 3; iteration++) {
        // simulate disconnects
        for (DiscoveryNode node : allNodes) {
            if (randomBoolean()) {
                final long disconnectionTime = randomLongBetween(0, 120000);
                maxDisconnectionTime = Math.max(maxDisconnectionTime, disconnectionTime);
                deterministicTaskQueue.scheduleAt(disconnectionTime, new Runnable() {

                    @Override
                    public void run() {
                        transportService.disconnectFromNode(node);
                    }

                    @Override
                    public String toString() {
                        return "scheduled disconnection of " + node;
                    }
                });
            }
        }
    }
    runTasksUntil(deterministicTaskQueue, maxDisconnectionTime);
    // disable exceptions so things can be restored
    transport.randomConnectionExceptions = false;
    logger.info("renewing connections");
    runTasksUntil(deterministicTaskQueue, maxDisconnectionTime + reconnectIntervalMillis);
    assertConnectedExactlyToNodes(transportService, targetNodes);
}

Example 2 with DeterministicTaskQueue

use of org.opensearch.cluster.coordination.DeterministicTaskQueue in project OpenSearch by opensearch-project.

the class NodeConnectionsServiceTests method testDebugLogging.

@TestLogging(reason = "testing that DEBUG-level logging is reasonable", value = "org.opensearch.cluster.NodeConnectionsService:DEBUG")
public void testDebugLogging() throws IllegalAccessException {
    final DeterministicTaskQueue deterministicTaskQueue = new DeterministicTaskQueue(builder().put(NODE_NAME_SETTING.getKey(), "node").build(), random());
    MockTransport transport = new MockTransport(deterministicTaskQueue.getThreadPool());
    TestTransportService transportService = new TestTransportService(transport, deterministicTaskQueue.getThreadPool());
    transportService.start();
    transportService.acceptIncomingRequests();
    final NodeConnectionsService service = new NodeConnectionsService(Settings.EMPTY, deterministicTaskQueue.getThreadPool(), transportService);
    service.start();
    final List<DiscoveryNode> allNodes = generateNodes();
    final DiscoveryNodes targetNodes = discoveryNodesFromList(randomSubsetOf(allNodes));
    service.connectToNodes(targetNodes, () -> {
    });
    deterministicTaskQueue.runAllRunnableTasks();
    // periodic reconnections to unexpectedly-disconnected nodes are logged
    final Set<DiscoveryNode> disconnectedNodes = new HashSet<>(randomSubsetOf(allNodes));
    for (DiscoveryNode disconnectedNode : disconnectedNodes) {
        transportService.disconnectFromNode(disconnectedNode);
    }
    final Logger logger = LogManager.getLogger("org.opensearch.cluster.NodeConnectionsService");
    try (MockLogAppender appender = MockLogAppender.createForLoggers(logger)) {
        for (DiscoveryNode targetNode : targetNodes) {
            if (disconnectedNodes.contains(targetNode)) {
                appender.addExpectation(new MockLogAppender.SeenEventExpectation("connecting to " + targetNode, "org.opensearch.cluster.NodeConnectionsService", Level.DEBUG, "connecting to " + targetNode));
                appender.addExpectation(new MockLogAppender.SeenEventExpectation("connected to " + targetNode, "org.opensearch.cluster.NodeConnectionsService", Level.DEBUG, "connected to " + targetNode));
            } else {
                appender.addExpectation(new MockLogAppender.UnseenEventExpectation("connecting to " + targetNode, "org.opensearch.cluster.NodeConnectionsService", Level.DEBUG, "connecting to " + targetNode));
                appender.addExpectation(new MockLogAppender.UnseenEventExpectation("connected to " + targetNode, "org.opensearch.cluster.NodeConnectionsService", Level.DEBUG, "connected to " + targetNode));
            }
        }
        runTasksUntil(deterministicTaskQueue, CLUSTER_NODE_RECONNECT_INTERVAL_SETTING.get(Settings.EMPTY).millis());
        appender.assertAllExpectationsMatched();
    }
    for (DiscoveryNode disconnectedNode : disconnectedNodes) {
        transportService.disconnectFromNode(disconnectedNode);
    }
    // changes to the expected set of nodes are logged, including reconnections to any unexpectedly-disconnected nodes
    final DiscoveryNodes newTargetNodes = discoveryNodesFromList(randomSubsetOf(allNodes));
    for (DiscoveryNode disconnectedNode : disconnectedNodes) {
        transportService.disconnectFromNode(disconnectedNode);
    }
    try (MockLogAppender appender = MockLogAppender.createForLoggers(logger)) {
        for (DiscoveryNode targetNode : targetNodes) {
            if (disconnectedNodes.contains(targetNode) && newTargetNodes.get(targetNode.getId()) != null) {
                appender.addExpectation(new MockLogAppender.SeenEventExpectation("connecting to " + targetNode, "org.opensearch.cluster.NodeConnectionsService", Level.DEBUG, "connecting to " + targetNode));
                appender.addExpectation(new MockLogAppender.SeenEventExpectation("connected to " + targetNode, "org.opensearch.cluster.NodeConnectionsService", Level.DEBUG, "connected to " + targetNode));
            } else {
                appender.addExpectation(new MockLogAppender.UnseenEventExpectation("connecting to " + targetNode, "org.opensearch.cluster.NodeConnectionsService", Level.DEBUG, "connecting to " + targetNode));
                appender.addExpectation(new MockLogAppender.UnseenEventExpectation("connected to " + targetNode, "org.opensearch.cluster.NodeConnectionsService", Level.DEBUG, "connected to " + targetNode));
            }
            if (newTargetNodes.get(targetNode.getId()) == null) {
                appender.addExpectation(new MockLogAppender.SeenEventExpectation("disconnected from " + targetNode, "org.opensearch.cluster.NodeConnectionsService", Level.DEBUG, "disconnected from " + targetNode));
            }
        }
        for (DiscoveryNode targetNode : newTargetNodes) {
            appender.addExpectation(new MockLogAppender.UnseenEventExpectation("disconnected from " + targetNode, "org.opensearch.cluster.NodeConnectionsService", Level.DEBUG, "disconnected from " + targetNode));
            if (targetNodes.get(targetNode.getId()) == null) {
                appender.addExpectation(new MockLogAppender.SeenEventExpectation("connecting to " + targetNode, "org.opensearch.cluster.NodeConnectionsService", Level.DEBUG, "connecting to " + targetNode));
                appender.addExpectation(new MockLogAppender.SeenEventExpectation("connected to " + targetNode, "org.opensearch.cluster.NodeConnectionsService", Level.DEBUG, "connected to " + targetNode));
            }
        }
        service.disconnectFromNodesExcept(newTargetNodes);
        service.connectToNodes(newTargetNodes, () -> {
        });
        deterministicTaskQueue.runAllRunnableTasks();
        appender.assertAllExpectationsMatched();
    }
}

Example 3 with DeterministicTaskQueue

use of org.opensearch.cluster.coordination.DeterministicTaskQueue in project OpenSearch by opensearch-project.

the class SnapshotResiliencyTests method createServices.

@Before
public void createServices() {
    tempDir = createTempDir();
    if (randomBoolean()) {
        blobStoreContext = new MockEventuallyConsistentRepository.Context();
    }
    deterministicTaskQueue = new DeterministicTaskQueue(Settings.builder().put(NODE_NAME_SETTING.getKey(), "shared").build(), random());
}

Example 4 with DeterministicTaskQueue

use of org.opensearch.cluster.coordination.DeterministicTaskQueue in project OpenSearch by opensearch-project.

the class TransportServiceDeserializationFailureTests method testDeserializationFailureLogIdentifiesListener.

public void testDeserializationFailureLogIdentifiesListener() {
    final DiscoveryNode localNode = new DiscoveryNode("local", buildNewFakeTransportAddress(), Version.CURRENT);
    final DiscoveryNode otherNode = new DiscoveryNode("other", buildNewFakeTransportAddress(), Version.CURRENT);
    final Settings settings = Settings.builder().put(NODE_NAME_SETTING.getKey(), "local").build();
    final DeterministicTaskQueue deterministicTaskQueue = new DeterministicTaskQueue(settings, random());
    final String testActionName = "internal:test-action";
    final MockTransport transport = new MockTransport() {

        @Override
        protected void onSendRequest(long requestId, String action, TransportRequest request, DiscoveryNode node) {
            if (action.equals(TransportService.HANDSHAKE_ACTION_NAME)) {
                handleResponse(requestId, new TransportService.HandshakeResponse(otherNode, new ClusterName(""), Version.CURRENT));
            }
        }
    };
    final TransportService transportService = transport.createTransportService(Settings.EMPTY, deterministicTaskQueue.getThreadPool(), TransportService.NOOP_TRANSPORT_INTERCEPTOR, ignored -> localNode, null, Collections.emptySet());
    transportService.registerRequestHandler(testActionName, ThreadPool.Names.SAME, TransportRequest.Empty::new, (request, channel, task) -> channel.sendResponse(TransportResponse.Empty.INSTANCE));
    transportService.start();
    transportService.acceptIncomingRequests();
    final PlainActionFuture<Void> connectionFuture = new PlainActionFuture<>();
    transportService.connectToNode(otherNode, connectionFuture);
    assertTrue(connectionFuture.isDone());
    {
        // requests without a parent task are recorded directly in the response context
        transportService.sendRequest(otherNode, testActionName, TransportRequest.Empty.INSTANCE, TransportRequestOptions.EMPTY, new TransportResponseHandler<TransportResponse.Empty>() {

            @Override
            public void handleResponse(TransportResponse.Empty response) {
                fail("should not be called");
            }

            @Override
            public void handleException(TransportException exp) {
                fail("should not be called");
            }

            @Override
            public String executor() {
                return ThreadPool.Names.SAME;
            }

            @Override
            public TransportResponse.Empty read(StreamInput in) {
                throw new AssertionError("should not be called");
            }

            @Override
            public String toString() {
                return "test handler without parent";
            }
        });
        final List<Transport.ResponseContext<? extends TransportResponse>> responseContexts = transport.getResponseHandlers().prune(ignored -> true);
        assertThat(responseContexts, hasSize(1));
        final TransportResponseHandler<? extends TransportResponse> handler = responseContexts.get(0).handler();
        assertThat(handler, hasToString(containsString("test handler without parent")));
    }
    {
        // requests with a parent task get wrapped up by the transport service, including the action name
        final Task parentTask = transportService.getTaskManager().register("test", "test-action", new TaskAwareRequest() {

            @Override
            public void setParentTask(TaskId taskId) {
                fail("should not be called");
            }

            @Override
            public TaskId getParentTask() {
                return TaskId.EMPTY_TASK_ID;
            }
        });
        transportService.sendChildRequest(otherNode, testActionName, TransportRequest.Empty.INSTANCE, parentTask, TransportRequestOptions.EMPTY, new TransportResponseHandler<TransportResponse.Empty>() {

            @Override
            public void handleResponse(TransportResponse.Empty response) {
                fail("should not be called");
            }

            @Override
            public void handleException(TransportException exp) {
                fail("should not be called");
            }

            @Override
            public String executor() {
                return ThreadPool.Names.SAME;
            }

            @Override
            public TransportResponse.Empty read(StreamInput in) {
                throw new AssertionError("should not be called");
            }

            @Override
            public String toString() {
                return "test handler with parent";
            }
        });
        final List<Transport.ResponseContext<? extends TransportResponse>> responseContexts = transport.getResponseHandlers().prune(ignored -> true);
        assertThat(responseContexts, hasSize(1));
        final TransportResponseHandler<? extends TransportResponse> handler = responseContexts.get(0).handler();
        assertThat(handler, hasToString(allOf(containsString("test handler with parent"), containsString(testActionName))));
    }
}

Example 5 with DeterministicTaskQueue

use of org.opensearch.cluster.coordination.DeterministicTaskQueue in project OpenSearch by opensearch-project.

the class DisruptableMockTransportTests method initTransports.

@Before
public void initTransports() {
    node1 = new DiscoveryNode("node1", buildNewFakeTransportAddress(), Version.CURRENT);
    node2 = new DiscoveryNode("node2", buildNewFakeTransportAddress(), Version.CURRENT);
    disconnectedLinks = new HashSet<>();
    blackholedLinks = new HashSet<>();
    blackholedRequestLinks = new HashSet<>();
    List<DisruptableMockTransport> transports = new ArrayList<>();
    deterministicTaskQueue = new DeterministicTaskQueue(Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), "dummy").build(), random());
    final DisruptableMockTransport transport1 = new DisruptableMockTransport(node1, logger, deterministicTaskQueue) {

        @Override
        protected ConnectionStatus getConnectionStatus(DiscoveryNode destination) {
            return DisruptableMockTransportTests.this.getConnectionStatus(getLocalNode(), destination);
        }

        @Override
        protected Optional<DisruptableMockTransport> getDisruptableMockTransport(TransportAddress address) {
            return transports.stream().filter(t -> t.getLocalNode().getAddress().equals(address)).findAny();
        }

        @Override
        protected void execute(Runnable runnable) {
            deterministicTaskQueue.scheduleNow(runnable);
        }
    };
    final DisruptableMockTransport transport2 = new DisruptableMockTransport(node2, logger, deterministicTaskQueue) {

        @Override
        protected ConnectionStatus getConnectionStatus(DiscoveryNode destination) {
            return DisruptableMockTransportTests.this.getConnectionStatus(getLocalNode(), destination);
        }

        @Override
        protected Optional<DisruptableMockTransport> getDisruptableMockTransport(TransportAddress address) {
            return transports.stream().filter(t -> t.getLocalNode().getAddress().equals(address)).findAny();
        }

        @Override
        protected void execute(Runnable runnable) {
            deterministicTaskQueue.scheduleNow(runnable);
        }
    };
    transports.add(transport1);
    transports.add(transport2);
    service1 = transport1.createTransportService(Settings.EMPTY, deterministicTaskQueue.getThreadPool(), NOOP_TRANSPORT_INTERCEPTOR, a -> node1, null, Collections.emptySet());
    service2 = transport2.createTransportService(Settings.EMPTY, deterministicTaskQueue.getThreadPool(), NOOP_TRANSPORT_INTERCEPTOR, a -> node2, null, Collections.emptySet());
    service1.start();
    service2.start();
    final PlainActionFuture<Void> fut1 = new PlainActionFuture<>();
    service1.connectToNode(node2, fut1);
    final PlainActionFuture<Void> fut2 = new PlainActionFuture<>();
    service2.connectToNode(node1, fut2);
    deterministicTaskQueue.runAllTasksInTimeOrder();
    assertTrue(fut1.isDone());
    assertTrue(fut2.isDone());
}