Examples with TransportService org.elasticsearch.transport.TransportService used on opensource projects

Example 76 with TransportService

use of org.elasticsearch.transport.TransportService in project crate by crate.

the class InternalTestCluster method reset.

private synchronized void reset(boolean wipeData) throws IOException {
    // clear all rules for mock transport services
    for (NodeAndClient nodeAndClient : nodes.values()) {
        TransportService transportService = nodeAndClient.node.injector().getInstance(TransportService.class);
        if (transportService instanceof MockTransportService) {
            final MockTransportService mockTransportService = (MockTransportService) transportService;
            mockTransportService.clearAllRules();
        }
    }
    randomlyResetClients();
    final int newSize = sharedNodesSeeds.length;
    if (nextNodeId.get() == newSize && nodes.size() == newSize) {
        if (wipeData) {
            wipePendingDataDirectories();
        }
        logger.debug("Cluster hasn't changed - moving out - nodes: [{}] nextNodeId: [{}] numSharedNodes: [{}]", nodes.keySet(), nextNodeId.get(), newSize);
        return;
    }
    logger.debug("Cluster is NOT consistent - restarting shared nodes - nodes: [{}] nextNodeId: [{}] numSharedNodes: [{}]", nodes.keySet(), nextNodeId.get(), newSize);
    // trash all nodes with id >= sharedNodesSeeds.length - they are non shared
    final List<NodeAndClient> toClose = new ArrayList<>();
    for (NodeAndClient nodeAndClient : nodes.values()) {
        if (nodeAndClient.nodeAndClientId() >= sharedNodesSeeds.length) {
            logger.debug("Close Node [{}] not shared", nodeAndClient.name);
            toClose.add(nodeAndClient);
        }
    }
    stopNodesAndClients(toClose);
    // clean up what the nodes left that is unused
    if (wipeData) {
        wipePendingDataDirectories();
    }
    assertTrue("expected at least one master-eligible node left in " + nodes, nodes.isEmpty() || nodes.values().stream().anyMatch(NodeAndClient::isMasterEligible));
    final int prevNodeCount = nodes.size();
    // start any missing node
    assert newSize == numSharedDedicatedMasterNodes + numSharedDataNodes + numSharedCoordOnlyNodes;
    // we want to start nodes in one go
    final List<NodeAndClient> toStartAndPublish = new ArrayList<>();
    final Runnable onTransportServiceStarted = () -> rebuildUnicastHostFiles(toStartAndPublish);
    final List<Settings> settings = new ArrayList<>();
    for (int i = 0; i < numSharedDedicatedMasterNodes; i++) {
        final Settings.Builder extraSettings = Settings.builder();
        extraSettings.put(Node.NODE_MASTER_SETTING.getKey(), true);
        extraSettings.put(Node.NODE_DATA_SETTING.getKey(), false);
        settings.add(getNodeSettings(i, sharedNodesSeeds[i], extraSettings.build()));
    }
    for (int i = numSharedDedicatedMasterNodes; i < numSharedDedicatedMasterNodes + numSharedDataNodes; i++) {
        final Settings.Builder extraSettings = Settings.builder();
        if (numSharedDedicatedMasterNodes > 0) {
            // if we don't have dedicated master nodes, keep things default
            extraSettings.put(Node.NODE_MASTER_SETTING.getKey(), false).build();
            extraSettings.put(Node.NODE_DATA_SETTING.getKey(), true).build();
        }
        settings.add(getNodeSettings(i, sharedNodesSeeds[i], extraSettings.build()));
    }
    for (int i = numSharedDedicatedMasterNodes + numSharedDataNodes; i < numSharedDedicatedMasterNodes + numSharedDataNodes + numSharedCoordOnlyNodes; i++) {
        final Builder extraSettings = Settings.builder().put(Node.NODE_MASTER_SETTING.getKey(), false).put(Node.NODE_DATA_SETTING.getKey(), false);
        settings.add(getNodeSettings(i, sharedNodesSeeds[i], extraSettings.build()));
    }
    int autoBootstrapMasterNodeIndex = -1;
    final List<String> masterNodeNames = settings.stream().filter(Node.NODE_MASTER_SETTING::get).map(Node.NODE_NAME_SETTING::get).collect(Collectors.toList());
    if (prevNodeCount == 0 && autoManageMasterNodes) {
        if (numSharedDedicatedMasterNodes > 0) {
            autoBootstrapMasterNodeIndex = RandomNumbers.randomIntBetween(random, 0, numSharedDedicatedMasterNodes - 1);
        } else if (numSharedDataNodes > 0) {
            autoBootstrapMasterNodeIndex = RandomNumbers.randomIntBetween(random, 0, numSharedDataNodes - 1);
        }
    }
    final List<Settings> updatedSettings = bootstrapMasterNodeWithSpecifiedIndex(settings);
    for (int i = 0; i < numSharedDedicatedMasterNodes + numSharedDataNodes + numSharedCoordOnlyNodes; i++) {
        Settings nodeSettings = updatedSettings.get(i);
        if (i == autoBootstrapMasterNodeIndex) {
            nodeSettings = Settings.builder().putList(INITIAL_MASTER_NODES_SETTING.getKey(), masterNodeNames).put(nodeSettings).build();
        }
        final NodeAndClient nodeAndClient = buildNode(i, nodeSettings, true, onTransportServiceStarted);
        toStartAndPublish.add(nodeAndClient);
    }
    startAndPublishNodesAndClients(toStartAndPublish);
    nextNodeId.set(newSize);
    assert size() == newSize;
    if (autoManageMasterNodes && newSize > 0) {
        validateClusterFormed();
    }
    logger.debug("Cluster is consistent again - nodes: [{}] nextNodeId: [{}] numSharedNodes: [{}]", nodes.keySet(), nextNodeId.get(), newSize);
}

Example 77 with TransportService

use of org.elasticsearch.transport.TransportService in project crate by crate.

the class JoinHelperTests method testJoinDeduplication.

public void testJoinDeduplication() {
    DeterministicTaskQueue deterministicTaskQueue = new DeterministicTaskQueue(Settings.builder().put(NODE_NAME_SETTING.getKey(), "node0").build(), random());
    CapturingTransport capturingTransport = new CapturingTransport();
    DiscoveryNode localNode = new DiscoveryNode("node0", buildNewFakeTransportAddress(), Version.CURRENT);
    TransportService transportService = capturingTransport.createTransportService(Settings.EMPTY, deterministicTaskQueue.getThreadPool(), x -> localNode, null);
    JoinHelper joinHelper = new JoinHelper(Settings.EMPTY, null, null, transportService, () -> 0L, () -> null, (joinRequest, joinCallback) -> {
        throw new AssertionError();
    }, startJoinRequest -> {
        throw new AssertionError();
    }, Collections.emptyList(), (s, p, r) -> {
    });
    transportService.start();
    DiscoveryNode node1 = new DiscoveryNode("node1", buildNewFakeTransportAddress(), Version.CURRENT);
    DiscoveryNode node2 = new DiscoveryNode("node2", buildNewFakeTransportAddress(), Version.CURRENT);
    assertFalse(joinHelper.isJoinPending());
    // check that sending a join to node1 works
    Optional<Join> optionalJoin1 = randomBoolean() ? Optional.empty() : Optional.of(new Join(localNode, node1, randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong()));
    joinHelper.sendJoinRequest(node1, optionalJoin1);
    CapturedRequest[] capturedRequests1 = capturingTransport.getCapturedRequestsAndClear();
    assertThat(capturedRequests1.length, equalTo(1));
    CapturedRequest capturedRequest1 = capturedRequests1[0];
    assertEquals(node1, capturedRequest1.node);
    assertTrue(joinHelper.isJoinPending());
    // check that sending a join to node2 works
    Optional<Join> optionalJoin2 = randomBoolean() ? Optional.empty() : Optional.of(new Join(localNode, node2, randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong()));
    joinHelper.sendJoinRequest(node2, optionalJoin2);
    CapturedRequest[] capturedRequests2 = capturingTransport.getCapturedRequestsAndClear();
    assertThat(capturedRequests2.length, equalTo(1));
    CapturedRequest capturedRequest2 = capturedRequests2[0];
    assertEquals(node2, capturedRequest2.node);
    // check that sending another join to node1 is a noop as the previous join is still in progress
    joinHelper.sendJoinRequest(node1, optionalJoin1);
    assertThat(capturingTransport.getCapturedRequestsAndClear().length, equalTo(0));
    // complete the previous join to node1
    if (randomBoolean()) {
        capturingTransport.handleResponse(capturedRequest1.requestId, TransportResponse.Empty.INSTANCE);
    } else {
        capturingTransport.handleRemoteError(capturedRequest1.requestId, new CoordinationStateRejectedException("dummy"));
    }
    // check that sending another join to node1 now works again
    joinHelper.sendJoinRequest(node1, optionalJoin1);
    CapturedRequest[] capturedRequests1a = capturingTransport.getCapturedRequestsAndClear();
    assertThat(capturedRequests1a.length, equalTo(1));
    CapturedRequest capturedRequest1a = capturedRequests1a[0];
    assertEquals(node1, capturedRequest1a.node);
    // check that sending another join to node2 works if the optionalJoin is different
    Optional<Join> optionalJoin2a = optionalJoin2.isPresent() && randomBoolean() ? Optional.empty() : Optional.of(new Join(localNode, node2, randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong()));
    joinHelper.sendJoinRequest(node2, optionalJoin2a);
    CapturedRequest[] capturedRequests2a = capturingTransport.getCapturedRequestsAndClear();
    assertThat(capturedRequests2a.length, equalTo(1));
    CapturedRequest capturedRequest2a = capturedRequests2a[0];
    assertEquals(node2, capturedRequest2a.node);
    // complete all the joins and check that isJoinPending is updated
    assertTrue(joinHelper.isJoinPending());
    capturingTransport.handleRemoteError(capturedRequest2.requestId, new CoordinationStateRejectedException("dummy"));
    capturingTransport.handleRemoteError(capturedRequest1a.requestId, new CoordinationStateRejectedException("dummy"));
    capturingTransport.handleRemoteError(capturedRequest2a.requestId, new CoordinationStateRejectedException("dummy"));
    assertFalse(joinHelper.isJoinPending());
}

Example 78 with TransportService

use of org.elasticsearch.transport.TransportService in project crate by crate.

the class GatewayMetaStatePersistedStateTests method testDataOnlyNodePersistence.

public void testDataOnlyNodePersistence() throws Exception {
    DiscoveryNode localNode = new DiscoveryNode("node1", buildNewFakeTransportAddress(), Collections.emptyMap(), Set.of(DiscoveryNodeRole.DATA_ROLE), Version.CURRENT);
    Settings settings = Settings.builder().put(ClusterName.CLUSTER_NAME_SETTING.getKey(), clusterName.value()).put(Node.NODE_MASTER_SETTING.getKey(), false).put(Node.NODE_NAME_SETTING.getKey(), "test").build();
    final MockGatewayMetaState gateway = new MockGatewayMetaState(localNode);
    final TransportService transportService = mock(TransportService.class);
    TestThreadPool threadPool = new TestThreadPool("testMarkAcceptedConfigAsCommittedOnDataOnlyNode");
    when(transportService.getThreadPool()).thenReturn(threadPool);
    ClusterService clusterService = mock(ClusterService.class);
    when(clusterService.getClusterSettings()).thenReturn(new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
    final PersistedClusterStateService persistedClusterStateService = new PersistedClusterStateService(nodeEnvironment, xContentRegistry(), BigArrays.NON_RECYCLING_INSTANCE, new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), () -> 0L);
    gateway.start(settings, transportService, clusterService, new MetaStateService(nodeEnvironment, xContentRegistry()), null, null, persistedClusterStateService);
    final CoordinationState.PersistedState persistedState = gateway.getPersistedState();
    assertThat(persistedState, instanceOf(GatewayMetaState.AsyncLucenePersistedState.class));
    // generate random coordinationMetadata with different lastAcceptedConfiguration and lastCommittedConfiguration
    CoordinationMetadata coordinationMetadata;
    do {
        coordinationMetadata = createCoordinationMetadata(randomNonNegativeLong());
    } while (coordinationMetadata.getLastAcceptedConfiguration().equals(coordinationMetadata.getLastCommittedConfiguration()));
    ClusterState state = createClusterState(randomNonNegativeLong(), Metadata.builder().coordinationMetadata(coordinationMetadata).clusterUUID(randomAlphaOfLength(10)).build());
    persistedState.setLastAcceptedState(state);
    assertBusy(() -> assertTrue(gateway.allPendingAsyncStatesWritten()));
    assertThat(persistedState.getLastAcceptedState().getLastAcceptedConfiguration(), not(equalTo(persistedState.getLastAcceptedState().getLastCommittedConfiguration())));
    CoordinationMetadata persistedCoordinationMetadata = persistedClusterStateService.loadBestOnDiskState().metadata.coordinationMetadata();
    assertThat(persistedCoordinationMetadata.getLastAcceptedConfiguration(), equalTo(GatewayMetaState.AsyncLucenePersistedState.STALE_STATE_CONFIG));
    assertThat(persistedCoordinationMetadata.getLastCommittedConfiguration(), equalTo(GatewayMetaState.AsyncLucenePersistedState.STALE_STATE_CONFIG));
    persistedState.markLastAcceptedStateAsCommitted();
    assertBusy(() -> assertTrue(gateway.allPendingAsyncStatesWritten()));
    CoordinationMetadata expectedCoordinationMetadata = CoordinationMetadata.builder(coordinationMetadata).lastCommittedConfiguration(coordinationMetadata.getLastAcceptedConfiguration()).build();
    ClusterState expectedClusterState = ClusterState.builder(state).metadata(Metadata.builder().coordinationMetadata(expectedCoordinationMetadata).clusterUUID(state.metadata().clusterUUID()).clusterUUIDCommitted(true).build()).build();
    assertClusterStateEqual(expectedClusterState, persistedState.getLastAcceptedState());
    persistedCoordinationMetadata = persistedClusterStateService.loadBestOnDiskState().metadata.coordinationMetadata();
    assertThat(persistedCoordinationMetadata.getLastAcceptedConfiguration(), equalTo(GatewayMetaState.AsyncLucenePersistedState.STALE_STATE_CONFIG));
    assertThat(persistedCoordinationMetadata.getLastCommittedConfiguration(), equalTo(GatewayMetaState.AsyncLucenePersistedState.STALE_STATE_CONFIG));
    assertTrue(persistedClusterStateService.loadBestOnDiskState().metadata.clusterUUIDCommitted());
    // generate a series of updates and check if batching works
    final String indexName = randomAlphaOfLength(10);
    long currentTerm = state.term();
    for (int i = 0; i < 1000; i++) {
        if (rarely()) {
            // bump term
            currentTerm = currentTerm + (rarely() ? randomIntBetween(1, 5) : 0L);
            persistedState.setCurrentTerm(currentTerm);
        } else {
            // update cluster state
            final int numberOfShards = randomIntBetween(1, 5);
            final long term = Math.min(state.term() + (rarely() ? randomIntBetween(1, 5) : 0L), currentTerm);
            final IndexMetadata indexMetadata = createIndexMetadata(indexName, numberOfShards, i);
            state = createClusterState(state.version() + 1, Metadata.builder().coordinationMetadata(createCoordinationMetadata(term)).put(indexMetadata, false).build());
            persistedState.setLastAcceptedState(state);
        }
    }
    assertEquals(currentTerm, persistedState.getCurrentTerm());
    assertClusterStateEqual(state, persistedState.getLastAcceptedState());
    assertBusy(() -> assertTrue(gateway.allPendingAsyncStatesWritten()));
    gateway.close();
    try (CoordinationState.PersistedState reloadedPersistedState = newGatewayPersistedState()) {
        assertEquals(currentTerm, reloadedPersistedState.getCurrentTerm());
        assertClusterStateEqual(GatewayMetaState.AsyncLucenePersistedState.resetVotingConfiguration(state), reloadedPersistedState.getLastAcceptedState());
        assertNotNull(reloadedPersistedState.getLastAcceptedState().metadata().index(indexName));
    }
    ThreadPool.terminate(threadPool, 10, TimeUnit.SECONDS);
}

Example 79 with TransportService

use of org.elasticsearch.transport.TransportService in project crate by crate.

the class MockTransport method createTransportService.

public TransportService createTransportService(Settings settings, ThreadPool threadPool, Function<BoundTransportAddress, DiscoveryNode> localNodeFactory, @Nullable ClusterSettings clusterSettings) {
    StubbableConnectionManager connectionManager = new StubbableConnectionManager(new ConnectionManager(settings, this), settings, this, threadPool);
    connectionManager.setDefaultNodeConnectedBehavior(cm -> Collections.emptySet());
    connectionManager.setDefaultGetConnectionBehavior((cm, discoveryNode) -> createConnection(discoveryNode));
    return new TransportService(settings, this, threadPool, localNodeFactory, clusterSettings, connectionManager);
}