Examples with TransportService org.elasticsearch.transport.TransportService used on opensource projects

Example 21 with TransportService

use of org.elasticsearch.transport.TransportService in project elasticsearch by elastic.

the class RemoteClusterService method updateRemoteClusters.

/**
     * This method updates the list of remote clusters. It's intended to be used as an update consumer on the settings infrastructure
     * @param seeds a cluster alias to discovery node mapping representing the remote clusters seeds nodes
     * @param connectionListener a listener invoked once every configured cluster has been connected to
     */
private synchronized void updateRemoteClusters(Map<String, List<DiscoveryNode>> seeds, ActionListener<Void> connectionListener) {
    if (seeds.containsKey(LOCAL_CLUSTER_GROUP_KEY)) {
        throw new IllegalArgumentException("remote clusters must not have the empty string as its key");
    }
    Map<String, RemoteClusterConnection> remoteClusters = new HashMap<>();
    if (seeds.isEmpty()) {
        connectionListener.onResponse(null);
    } else {
        CountDown countDown = new CountDown(seeds.size());
        Predicate<DiscoveryNode> nodePredicate = (node) -> Version.CURRENT.isCompatible(node.getVersion());
        if (REMOTE_NODE_ATTRIBUTE.exists(settings)) {
            // nodes can be tagged with node.attr.remote_gateway: true to allow a node to be a gateway node for
            // cross cluster search
            String attribute = REMOTE_NODE_ATTRIBUTE.get(settings);
            nodePredicate = nodePredicate.and((node) -> Boolean.getBoolean(node.getAttributes().getOrDefault(attribute, "false")));
        }
        remoteClusters.putAll(this.remoteClusters);
        for (Map.Entry<String, List<DiscoveryNode>> entry : seeds.entrySet()) {
            RemoteClusterConnection remote = this.remoteClusters.get(entry.getKey());
            if (entry.getValue().isEmpty()) {
                // with no seed nodes we just remove the connection
                try {
                    IOUtils.close(remote);
                } catch (IOException e) {
                    logger.warn("failed to close remote cluster connections for cluster: " + entry.getKey(), e);
                }
                remoteClusters.remove(entry.getKey());
                continue;
            }
            if (remote == null) {
                // this is a new cluster we have to add a new representation
                remote = new RemoteClusterConnection(settings, entry.getKey(), entry.getValue(), transportService, numRemoteConnections, nodePredicate);
                remoteClusters.put(entry.getKey(), remote);
            }
            // now update the seed nodes no matter if it's new or already existing
            RemoteClusterConnection finalRemote = remote;
            remote.updateSeedNodes(entry.getValue(), ActionListener.wrap(response -> {
                if (countDown.countDown()) {
                    connectionListener.onResponse(response);
                }
            }, exception -> {
                if (countDown.fastForward()) {
                    connectionListener.onFailure(exception);
                }
                if (finalRemote.isClosed() == false) {
                    logger.warn("failed to update seed list for cluster: " + entry.getKey(), exception);
                }
            }));
        }
    }
    this.remoteClusters = Collections.unmodifiableMap(remoteClusters);
}

Example 22 with TransportService

use of org.elasticsearch.transport.TransportService in project elasticsearch by elastic.

the class TransportClient method buildTemplate.

private static ClientTemplate buildTemplate(Settings providedSettings, Settings defaultSettings, Collection<Class<? extends Plugin>> plugins, HostFailureListener failureListner) {
    if (Node.NODE_NAME_SETTING.exists(providedSettings) == false) {
        providedSettings = Settings.builder().put(providedSettings).put(Node.NODE_NAME_SETTING.getKey(), "_client_").build();
    }
    final PluginsService pluginsService = newPluginService(providedSettings, plugins);
    final Settings settings = Settings.builder().put(defaultSettings).put(pluginsService.updatedSettings()).build();
    final List<Closeable> resourcesToClose = new ArrayList<>();
    final ThreadPool threadPool = new ThreadPool(settings);
    resourcesToClose.add(() -> ThreadPool.terminate(threadPool, 10, TimeUnit.SECONDS));
    final NetworkService networkService = new NetworkService(settings, Collections.emptyList());
    try {
        final List<Setting<?>> additionalSettings = new ArrayList<>();
        final List<String> additionalSettingsFilter = new ArrayList<>();
        additionalSettings.addAll(pluginsService.getPluginSettings());
        additionalSettingsFilter.addAll(pluginsService.getPluginSettingsFilter());
        for (final ExecutorBuilder<?> builder : threadPool.builders()) {
            additionalSettings.addAll(builder.getRegisteredSettings());
        }
        SettingsModule settingsModule = new SettingsModule(settings, additionalSettings, additionalSettingsFilter);
        SearchModule searchModule = new SearchModule(settings, true, pluginsService.filterPlugins(SearchPlugin.class));
        List<NamedWriteableRegistry.Entry> entries = new ArrayList<>();
        entries.addAll(NetworkModule.getNamedWriteables());
        entries.addAll(searchModule.getNamedWriteables());
        entries.addAll(ClusterModule.getNamedWriteables());
        entries.addAll(pluginsService.filterPlugins(Plugin.class).stream().flatMap(p -> p.getNamedWriteables().stream()).collect(Collectors.toList()));
        NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(entries);
        NamedXContentRegistry xContentRegistry = new NamedXContentRegistry(Stream.of(searchModule.getNamedXContents().stream(), pluginsService.filterPlugins(Plugin.class).stream().flatMap(p -> p.getNamedXContent().stream())).flatMap(Function.identity()).collect(toList()));
        ModulesBuilder modules = new ModulesBuilder();
        // plugin modules must be added here, before others or we can get crazy injection errors...
        for (Module pluginModule : pluginsService.createGuiceModules()) {
            modules.add(pluginModule);
        }
        modules.add(b -> b.bind(ThreadPool.class).toInstance(threadPool));
        ActionModule actionModule = new ActionModule(true, settings, null, settingsModule.getIndexScopedSettings(), settingsModule.getClusterSettings(), settingsModule.getSettingsFilter(), threadPool, pluginsService.filterPlugins(ActionPlugin.class), null, null);
        modules.add(actionModule);
        CircuitBreakerService circuitBreakerService = Node.createCircuitBreakerService(settingsModule.getSettings(), settingsModule.getClusterSettings());
        resourcesToClose.add(circuitBreakerService);
        BigArrays bigArrays = new BigArrays(settings, circuitBreakerService);
        resourcesToClose.add(bigArrays);
        modules.add(settingsModule);
        NetworkModule networkModule = new NetworkModule(settings, true, pluginsService.filterPlugins(NetworkPlugin.class), threadPool, bigArrays, circuitBreakerService, namedWriteableRegistry, xContentRegistry, networkService, null);
        final Transport transport = networkModule.getTransportSupplier().get();
        final TransportService transportService = new TransportService(settings, transport, threadPool, networkModule.getTransportInterceptor(), boundTransportAddress -> DiscoveryNode.createLocal(settings, new TransportAddress(TransportAddress.META_ADDRESS, 0), UUIDs.randomBase64UUID()), null);
        modules.add((b -> {
            b.bind(BigArrays.class).toInstance(bigArrays);
            b.bind(PluginsService.class).toInstance(pluginsService);
            b.bind(CircuitBreakerService.class).toInstance(circuitBreakerService);
            b.bind(NamedWriteableRegistry.class).toInstance(namedWriteableRegistry);
            b.bind(Transport.class).toInstance(transport);
            b.bind(TransportService.class).toInstance(transportService);
            b.bind(NetworkService.class).toInstance(networkService);
        }));
        Injector injector = modules.createInjector();
        final TransportClientNodesService nodesService = new TransportClientNodesService(settings, transportService, threadPool, failureListner == null ? (t, e) -> {
        } : failureListner);
        final TransportProxyClient proxy = new TransportProxyClient(settings, transportService, nodesService, actionModule.getActions().values().stream().map(x -> x.getAction()).collect(Collectors.toList()));
        List<LifecycleComponent> pluginLifecycleComponents = new ArrayList<>();
        pluginLifecycleComponents.addAll(pluginsService.getGuiceServiceClasses().stream().map(injector::getInstance).collect(Collectors.toList()));
        resourcesToClose.addAll(pluginLifecycleComponents);
        transportService.start();
        transportService.acceptIncomingRequests();
        ClientTemplate transportClient = new ClientTemplate(injector, pluginLifecycleComponents, nodesService, proxy, namedWriteableRegistry);
        resourcesToClose.clear();
        return transportClient;
    } finally {
        IOUtils.closeWhileHandlingException(resourcesToClose);
    }
}

Example 23 with TransportService

use of org.elasticsearch.transport.TransportService in project elasticsearch by elastic.

the class Node method start.

/**
     * Start the node. If the node is already started, this method is no-op.
     */
public Node start() throws NodeValidationException {
    if (!lifecycle.moveToStarted()) {
        return this;
    }
    Logger logger = Loggers.getLogger(Node.class, NODE_NAME_SETTING.get(settings));
    logger.info("starting ...");
    // hack around dependency injection problem (for now...)
    injector.getInstance(Discovery.class).setAllocationService(injector.getInstance(AllocationService.class));
    pluginLifecycleComponents.forEach(LifecycleComponent::start);
    injector.getInstance(MappingUpdatedAction.class).setClient(client);
    injector.getInstance(IndicesService.class).start();
    injector.getInstance(IndicesClusterStateService.class).start();
    injector.getInstance(SnapshotsService.class).start();
    injector.getInstance(SnapshotShardsService.class).start();
    injector.getInstance(RoutingService.class).start();
    injector.getInstance(SearchService.class).start();
    injector.getInstance(MonitorService.class).start();
    final ClusterService clusterService = injector.getInstance(ClusterService.class);
    final NodeConnectionsService nodeConnectionsService = injector.getInstance(NodeConnectionsService.class);
    nodeConnectionsService.start();
    clusterService.setNodeConnectionsService(nodeConnectionsService);
    // TODO hack around circular dependencies problems
    injector.getInstance(GatewayAllocator.class).setReallocation(clusterService, injector.getInstance(RoutingService.class));
    injector.getInstance(ResourceWatcherService.class).start();
    injector.getInstance(GatewayService.class).start();
    Discovery discovery = injector.getInstance(Discovery.class);
    clusterService.setDiscoverySettings(discovery.getDiscoverySettings());
    clusterService.addInitialStateBlock(discovery.getDiscoverySettings().getNoMasterBlock());
    clusterService.setClusterStatePublisher(discovery::publish);
    // start before the cluster service since it adds/removes initial Cluster state blocks
    final TribeService tribeService = injector.getInstance(TribeService.class);
    tribeService.start();
    // Start the transport service now so the publish address will be added to the local disco node in ClusterService
    TransportService transportService = injector.getInstance(TransportService.class);
    transportService.getTaskManager().setTaskResultsService(injector.getInstance(TaskResultsService.class));
    transportService.start();
    validateNodeBeforeAcceptingRequests(settings, transportService.boundAddress(), pluginsService.filterPlugins(Plugin.class).stream().flatMap(p -> p.getBootstrapChecks().stream()).collect(Collectors.toList()));
    clusterService.addStateApplier(transportService.getTaskManager());
    clusterService.start();
    assert localNodeFactory.getNode() != null;
    assert transportService.getLocalNode().equals(localNodeFactory.getNode()) : "transportService has a different local node than the factory provided";
    assert clusterService.localNode().equals(localNodeFactory.getNode()) : "clusterService has a different local node than the factory provided";
    // start after cluster service so the local disco is known
    discovery.start();
    transportService.acceptIncomingRequests();
    discovery.startInitialJoin();
    // tribe nodes don't have a master so we shouldn't register an observer         s
    final TimeValue initialStateTimeout = DiscoverySettings.INITIAL_STATE_TIMEOUT_SETTING.get(settings);
    if (initialStateTimeout.millis() > 0) {
        final ThreadPool thread = injector.getInstance(ThreadPool.class);
        ClusterState clusterState = clusterService.state();
        ClusterStateObserver observer = new ClusterStateObserver(clusterState, clusterService, null, logger, thread.getThreadContext());
        if (clusterState.nodes().getMasterNodeId() == null) {
            logger.debug("waiting to join the cluster. timeout [{}]", initialStateTimeout);
            final CountDownLatch latch = new CountDownLatch(1);
            observer.waitForNextChange(new ClusterStateObserver.Listener() {

                @Override
                public void onNewClusterState(ClusterState state) {
                    latch.countDown();
                }

                @Override
                public void onClusterServiceClose() {
                    latch.countDown();
                }

                @Override
                public void onTimeout(TimeValue timeout) {
                    logger.warn("timed out while waiting for initial discovery state - timeout: {}", initialStateTimeout);
                    latch.countDown();
                }
            }, state -> state.nodes().getMasterNodeId() != null, initialStateTimeout);
            try {
                latch.await();
            } catch (InterruptedException e) {
                throw new ElasticsearchTimeoutException("Interrupted while waiting for initial discovery state");
            }
        }
    }
    if (NetworkModule.HTTP_ENABLED.get(settings)) {
        injector.getInstance(HttpServerTransport.class).start();
    }
    // start nodes now, after the http server, because it may take some time
    tribeService.startNodes();
    // starts connecting to remote clusters if any cluster is configured
    SearchTransportService searchTransportService = injector.getInstance(SearchTransportService.class);
    searchTransportService.start();
    if (WRITE_PORTS_FIELD_SETTING.get(settings)) {
        if (NetworkModule.HTTP_ENABLED.get(settings)) {
            HttpServerTransport http = injector.getInstance(HttpServerTransport.class);
            writePortsFile("http", http.boundAddress());
        }
        TransportService transport = injector.getInstance(TransportService.class);
        writePortsFile("transport", transport.boundAddress());
    }
    logger.info("started");
    return this;
}

Example 24 with TransportService

use of org.elasticsearch.transport.TransportService in project elasticsearch by elastic.

the class TasksIT method testCanFetchIndexStatus.

/**
     * Very basic "is it plugged in" style test that indexes a document and makes sure that you can fetch the status of the process. The
     * goal here is to verify that the large moving parts that make fetching task status work fit together rather than to verify any
     * particular status results from indexing. For that, look at {@link TransportReplicationActionTests}. We intentionally don't use the
     * task recording mechanism used in other places in this test so we can make sure that the status fetching works properly over the wire.
     */
public void testCanFetchIndexStatus() throws Exception {
    // First latch waits for the task to start, second on blocks it from finishing.
    CountDownLatch taskRegistered = new CountDownLatch(1);
    CountDownLatch letTaskFinish = new CountDownLatch(1);
    Thread index = null;
    try {
        for (TransportService transportService : internalCluster().getInstances(TransportService.class)) {
            ((MockTaskManager) transportService.getTaskManager()).addListener(new MockTaskManagerListener() {

                @Override
                public void onTaskRegistered(Task task) {
                    if (task.getAction().startsWith(IndexAction.NAME)) {
                        taskRegistered.countDown();
                        logger.debug("Blocking [{}] starting", task);
                        try {
                            assertTrue(letTaskFinish.await(10, TimeUnit.SECONDS));
                        } catch (InterruptedException e) {
                            throw new RuntimeException(e);
                        }
                    }
                }

                @Override
                public void onTaskUnregistered(Task task) {
                }

                @Override
                public void waitForTaskCompletion(Task task) {
                }
            });
        }
        // Need to run the task in a separate thread because node client's .execute() is blocked by our task listener
        index = new Thread(() -> {
            IndexResponse indexResponse = client().prepareIndex("test", "test").setSource("test", "test").get();
            assertArrayEquals(ReplicationResponse.EMPTY, indexResponse.getShardInfo().getFailures());
        });
        index.start();
        // waiting for at least one task to be registered
        assertTrue(taskRegistered.await(10, TimeUnit.SECONDS));
        ListTasksResponse listResponse = client().admin().cluster().prepareListTasks().setActions("indices:data/write/index*").setDetailed(true).get();
        assertThat(listResponse.getTasks(), not(empty()));
        for (TaskInfo task : listResponse.getTasks()) {
            assertNotNull(task.getStatus());
            GetTaskResponse getResponse = client().admin().cluster().prepareGetTask(task.getTaskId()).get();
            assertFalse("task should still be running", getResponse.getTask().isCompleted());
            TaskInfo fetchedWithGet = getResponse.getTask().getTask();
            assertEquals(task.getId(), fetchedWithGet.getId());
            assertEquals(task.getType(), fetchedWithGet.getType());
            assertEquals(task.getAction(), fetchedWithGet.getAction());
            assertEquals(task.getDescription(), fetchedWithGet.getDescription());
            assertEquals(task.getStatus(), fetchedWithGet.getStatus());
            assertEquals(task.getStartTime(), fetchedWithGet.getStartTime());
            assertThat(fetchedWithGet.getRunningTimeNanos(), greaterThanOrEqualTo(task.getRunningTimeNanos()));
            assertEquals(task.isCancellable(), fetchedWithGet.isCancellable());
            assertEquals(task.getParentTaskId(), fetchedWithGet.getParentTaskId());
        }
    } finally {
        letTaskFinish.countDown();
        if (index != null) {
            index.join();
        }
        assertBusy(() -> {
            assertEquals(emptyList(), client().admin().cluster().prepareListTasks().setActions("indices:data/write/index*").get().getTasks());
        });
    }
}

Example 25 with TransportService

use of org.elasticsearch.transport.TransportService in project elasticsearch by elastic.

the class TasksIT method waitForCompletionTestCase.

/**
     * Test wait for completion.
     * @param storeResult should the task store its results
     * @param wait start waiting for a task. Accepts that id of the task to wait for and returns a future waiting for it.
     * @param validator validate the response and return the task ids that were found
     */
private <T> void waitForCompletionTestCase(boolean storeResult, Function<TaskId, ListenableActionFuture<T>> wait, Consumer<T> validator) throws Exception {
    // Start blocking test task
    ListenableActionFuture<TestTaskPlugin.NodesResponse> future = TestTaskPlugin.TestTaskAction.INSTANCE.newRequestBuilder(client()).setShouldStoreResult(storeResult).execute();
    ListenableActionFuture<T> waitResponseFuture;
    TaskId taskId;
    try {
        taskId = waitForTestTaskStartOnAllNodes();
        // Wait for the task to start
        assertBusy(() -> client().admin().cluster().prepareGetTask(taskId).get());
        // Register listeners so we can be sure the waiting started
        CountDownLatch waitForWaitingToStart = new CountDownLatch(1);
        for (TransportService transportService : internalCluster().getInstances(TransportService.class)) {
            ((MockTaskManager) transportService.getTaskManager()).addListener(new MockTaskManagerListener() {

                @Override
                public void waitForTaskCompletion(Task task) {
                    waitForWaitingToStart.countDown();
                }

                @Override
                public void onTaskRegistered(Task task) {
                }

                @Override
                public void onTaskUnregistered(Task task) {
                }
            });
        }
        // Spin up a request to wait for the test task to finish
        waitResponseFuture = wait.apply(taskId);
        /* Wait for the wait to start. This should count down just *before* we wait for completion but after the list/get has got a
             * reference to the running task. Because we unblock immediately after this the task may no longer be running for us to wait
             * on which is fine. */
        waitForWaitingToStart.await();
    } finally {
        // Unblock the request so the wait for completion request can finish
        TestTaskPlugin.UnblockTestTasksAction.INSTANCE.newRequestBuilder(client()).get();
    }
    // Now that the task is unblocked the list response will come back
    T waitResponse = waitResponseFuture.get();
    validator.accept(waitResponse);
    TestTaskPlugin.NodesResponse response = future.get();
    assertEquals(emptyList(), response.failures());
}