Examples with TransportService org.opensearch.transport.TransportService used on opensource projects

Example 41 with TransportService

use of org.opensearch.transport.TransportService in project OpenSearch by opensearch-project.

the class CancellableTasksIT method testCancelOrphanedTasks.

public void testCancelOrphanedTasks() throws Exception {
    final String nodeWithRootTask = internalCluster().startDataOnlyNode();
    Set<DiscoveryNode> nodes = StreamSupport.stream(clusterService().state().nodes().spliterator(), false).collect(Collectors.toSet());
    TestRequest rootRequest = generateTestRequest(nodes, 0, between(1, 3));
    client(nodeWithRootTask).execute(TransportTestAction.ACTION, rootRequest);
    allowPartialRequest(rootRequest);
    try {
        internalCluster().stopRandomNode(InternalTestCluster.nameFilter(nodeWithRootTask));
        assertBusy(() -> {
            for (TransportService transportService : internalCluster().getInstances(TransportService.class)) {
                for (CancellableTask task : transportService.getTaskManager().getCancellableTasks().values()) {
                    if (task.getAction().equals(TransportTestAction.ACTION.name())) {
                        final TaskInfo taskInfo = task.taskInfo(transportService.getLocalNode().getId(), false);
                        assertTrue(taskInfo.toString(), task.isCancelled());
                        assertNotNull(taskInfo.toString(), task.getReasonCancelled());
                        assertThat(taskInfo.toString(), task.getReasonCancelled(), equalTo("channel was closed"));
                    }
                }
            }
        }, 30, TimeUnit.SECONDS);
    } finally {
        allowEntireRequest(rootRequest);
        ensureAllBansRemoved();
    }
}

Example 42 with TransportService

use of org.opensearch.transport.TransportService in project OpenSearch by opensearch-project.

the class IndexRecoveryIT method testRecoverLocallyUpToGlobalCheckpoint.

public void testRecoverLocallyUpToGlobalCheckpoint() throws Exception {
    internalCluster().ensureAtLeastNumDataNodes(2);
    List<String> nodes = randomSubsetOf(2, StreamSupport.stream(clusterService().state().nodes().getDataNodes().spliterator(), false).map(node -> node.value.getName()).collect(Collectors.toSet()));
    String indexName = "test-index";
    createIndex(indexName, Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 1).put(IndexService.GLOBAL_CHECKPOINT_SYNC_INTERVAL_SETTING.getKey(), "12h").put("index.routing.allocation.include._name", String.join(",", nodes)).build());
    ensureGreen(indexName);
    int numDocs = randomIntBetween(0, 100);
    indexRandom(randomBoolean(), false, randomBoolean(), IntStream.range(0, numDocs).mapToObj(n -> client().prepareIndex(indexName).setSource("num", n)).collect(toList()));
    // avoid refresh when we are failing a shard
    client().admin().indices().prepareRefresh(indexName).get();
    String failingNode = randomFrom(nodes);
    PlainActionFuture<StartRecoveryRequest> startRecoveryRequestFuture = new PlainActionFuture<>();
    // Peer recovery fails if the primary does not see the recovering replica in the replication group (when the cluster state
    // update on the primary is delayed). To verify the local recovery stats, we have to manually remember this value in the
    // first try because the local recovery happens once and its stats is reset when the recovery fails.
    SetOnce<Integer> localRecoveredOps = new SetOnce<>();
    for (String node : nodes) {
        MockTransportService transportService = (MockTransportService) internalCluster().getInstance(TransportService.class, node);
        transportService.addSendBehavior((connection, requestId, action, request, options) -> {
            if (action.equals(PeerRecoverySourceService.Actions.START_RECOVERY)) {
                final RecoveryState recoveryState = internalCluster().getInstance(IndicesService.class, failingNode).getShardOrNull(new ShardId(resolveIndex(indexName), 0)).recoveryState();
                assertThat(recoveryState.getTranslog().recoveredOperations(), equalTo(recoveryState.getTranslog().totalLocal()));
                if (startRecoveryRequestFuture.isDone()) {
                    assertThat(recoveryState.getTranslog().totalLocal(), equalTo(0));
                    recoveryState.getTranslog().totalLocal(localRecoveredOps.get());
                    recoveryState.getTranslog().incrementRecoveredOperations(localRecoveredOps.get());
                } else {
                    localRecoveredOps.set(recoveryState.getTranslog().totalLocal());
                    startRecoveryRequestFuture.onResponse((StartRecoveryRequest) request);
                }
            }
            if (action.equals(PeerRecoveryTargetService.Actions.FILE_CHUNK)) {
                RetentionLeases retentionLeases = internalCluster().getInstance(IndicesService.class, node).indexServiceSafe(resolveIndex(indexName)).getShard(0).getRetentionLeases();
                throw new AssertionError("expect an operation-based recovery:" + "retention leases" + Strings.toString(retentionLeases) + "]");
            }
            connection.sendRequest(requestId, action, request, options);
        });
    }
    IndexShard shard = internalCluster().getInstance(IndicesService.class, failingNode).getShardOrNull(new ShardId(resolveIndex(indexName), 0));
    final long lastSyncedGlobalCheckpoint = shard.getLastSyncedGlobalCheckpoint();
    final long localCheckpointOfSafeCommit;
    try (Engine.IndexCommitRef safeCommitRef = shard.acquireSafeIndexCommit()) {
        localCheckpointOfSafeCommit = SequenceNumbers.loadSeqNoInfoFromLuceneCommit(safeCommitRef.getIndexCommit().getUserData().entrySet()).localCheckpoint;
    }
    final long maxSeqNo = shard.seqNoStats().getMaxSeqNo();
    shard.failShard("test", new IOException("simulated"));
    StartRecoveryRequest startRecoveryRequest = startRecoveryRequestFuture.actionGet();
    logger.info("--> start recovery request: starting seq_no {}, commit {}", startRecoveryRequest.startingSeqNo(), startRecoveryRequest.metadataSnapshot().getCommitUserData());
    SequenceNumbers.CommitInfo commitInfoAfterLocalRecovery = SequenceNumbers.loadSeqNoInfoFromLuceneCommit(startRecoveryRequest.metadataSnapshot().getCommitUserData().entrySet());
    assertThat(commitInfoAfterLocalRecovery.localCheckpoint, equalTo(lastSyncedGlobalCheckpoint));
    assertThat(commitInfoAfterLocalRecovery.maxSeqNo, equalTo(lastSyncedGlobalCheckpoint));
    assertThat(startRecoveryRequest.startingSeqNo(), equalTo(lastSyncedGlobalCheckpoint + 1));
    ensureGreen(indexName);
    assertThat((long) localRecoveredOps.get(), equalTo(lastSyncedGlobalCheckpoint - localCheckpointOfSafeCommit));
    for (RecoveryState recoveryState : client().admin().indices().prepareRecoveries().get().shardRecoveryStates().get(indexName)) {
        if (startRecoveryRequest.targetNode().equals(recoveryState.getTargetNode())) {
            assertThat("expect an operation-based recovery", recoveryState.getIndex().fileDetails(), empty());
            assertThat("total recovered translog operations must include both local and remote recovery", recoveryState.getTranslog().recoveredOperations(), greaterThanOrEqualTo(Math.toIntExact(maxSeqNo - localCheckpointOfSafeCommit)));
        }
    }
    for (String node : nodes) {
        MockTransportService transportService = (MockTransportService) internalCluster().getInstance(TransportService.class, node);
        transportService.clearAllRules();
    }
}

Example 43 with TransportService

use of org.opensearch.transport.TransportService in project OpenSearch by opensearch-project.

the class Ec2DiscoveryTests method buildDynamicHosts.

protected List<TransportAddress> buildDynamicHosts(Settings nodeSettings, int nodes, List<List<Tag>> tagsList) {
    final String accessKey = "ec2_key";
    try (Ec2DiscoveryPlugin plugin = new Ec2DiscoveryPlugin(buildSettings(accessKey))) {
        AwsEc2SeedHostsProvider provider = new AwsEc2SeedHostsProvider(nodeSettings, transportService, plugin.ec2Service);
        httpServer.createContext("/", exchange -> {
            if (exchange.getRequestMethod().equals(HttpMethodName.POST.name())) {
                final String request = Streams.readFully(exchange.getRequestBody()).toBytesRef().utf8ToString();
                final String userAgent = exchange.getRequestHeaders().getFirst("User-Agent");
                if (userAgent != null && userAgent.startsWith("aws-sdk-java")) {
                    final String auth = exchange.getRequestHeaders().getFirst("Authorization");
                    if (auth == null || auth.contains(accessKey) == false) {
                        throw new IllegalArgumentException("wrong access key: " + auth);
                    }
                    // Simulate an EC2 DescribeInstancesResponse
                    final Map<String, List<String>> tagsIncluded = new HashMap<>();
                    final String[] params = request.split("&");
                    Arrays.stream(params).filter(entry -> entry.startsWith("Filter.") && entry.contains("=tag%3A")).forEach(entry -> {
                        final int startIndex = "Filter.".length();
                        final int filterId = Integer.parseInt(entry.substring(startIndex, entry.indexOf(".", startIndex)));
                        tagsIncluded.put(entry.substring(entry.indexOf("=tag%3A") + "=tag%3A".length()), Arrays.stream(params).filter(param -> param.startsWith("Filter." + filterId + ".Value.")).map(param -> param.substring(param.indexOf("=") + 1)).collect(Collectors.toList()));
                    });
                    final List<Instance> instances = IntStream.range(1, nodes + 1).mapToObj(node -> {
                        final String instanceId = "node" + node;
                        final Instance instance = new Instance().withInstanceId(instanceId).withState(new InstanceState().withName(InstanceStateName.Running)).withPrivateDnsName(PREFIX_PRIVATE_DNS + instanceId + SUFFIX_PRIVATE_DNS).withPublicDnsName(PREFIX_PUBLIC_DNS + instanceId + SUFFIX_PUBLIC_DNS).withPrivateIpAddress(PREFIX_PRIVATE_IP + node).withPublicIpAddress(PREFIX_PUBLIC_IP + node);
                        if (tagsList != null) {
                            instance.setTags(tagsList.get(node - 1));
                        }
                        return instance;
                    }).filter(instance -> tagsIncluded.entrySet().stream().allMatch(entry -> instance.getTags().stream().filter(t -> t.getKey().equals(entry.getKey())).map(Tag::getValue).collect(Collectors.toList()).containsAll(entry.getValue()))).collect(Collectors.toList());
                    for (NameValuePair parse : URLEncodedUtils.parse(request, UTF_8)) {
                        if ("Action".equals(parse.getName())) {
                            final byte[] responseBody = generateDescribeInstancesResponse(instances);
                            exchange.getResponseHeaders().set("Content-Type", "text/xml; charset=UTF-8");
                            exchange.sendResponseHeaders(HttpStatus.SC_OK, responseBody.length);
                            exchange.getResponseBody().write(responseBody);
                            return;
                        }
                    }
                }
            }
            fail("did not send response");
        });
        List<TransportAddress> dynamicHosts = provider.getSeedAddresses(null);
        logger.debug("--> addresses found: {}", dynamicHosts);
        return dynamicHosts;
    } catch (IOException e) {
        fail("Unexpected IOException");
        return null;
    }
}

Example 44 with TransportService

use of org.opensearch.transport.TransportService in project OpenSearch by opensearch-project.

the class EC2RetriesTests method testEC2DiscoveryRetriesOnRateLimiting.

public void testEC2DiscoveryRetriesOnRateLimiting() throws IOException {
    final String accessKey = "ec2_access";
    final List<String> hosts = Collections.singletonList("127.0.0.1:9300");
    final Map<String, Integer> failedRequests = new ConcurrentHashMap<>();
    // retry the same request 5 times at most
    final int maxRetries = randomIntBetween(1, 5);
    httpServer.createContext("/", exchange -> {
        if (exchange.getRequestMethod().equals(HttpMethodName.POST.name())) {
            final String request = Streams.readFully(exchange.getRequestBody()).utf8ToString();
            final String userAgent = exchange.getRequestHeaders().getFirst("User-Agent");
            if (userAgent != null && userAgent.startsWith("aws-sdk-java")) {
                final String auth = exchange.getRequestHeaders().getFirst("Authorization");
                if (auth == null || auth.contains(accessKey) == false) {
                    throw new IllegalArgumentException("wrong access key: " + auth);
                }
                if (failedRequests.compute(exchange.getRequestHeaders().getFirst("Amz-sdk-invocation-id"), (requestId, count) -> (count == null ? 0 : count) + 1) < maxRetries) {
                    exchange.sendResponseHeaders(HttpStatus.SC_SERVICE_UNAVAILABLE, -1);
                    return;
                }
                // Simulate an EC2 DescribeInstancesResponse
                byte[] responseBody = null;
                for (NameValuePair parse : URLEncodedUtils.parse(request, UTF_8)) {
                    if ("Action".equals(parse.getName())) {
                        responseBody = generateDescribeInstancesResponse(hosts.stream().map(address -> new Instance().withPublicIpAddress(address)).collect(Collectors.toList()));
                        break;
                    }
                }
                responseBody = responseBody == null ? new byte[0] : responseBody;
                exchange.getResponseHeaders().set("Content-Type", "text/xml; charset=UTF-8");
                exchange.sendResponseHeaders(HttpStatus.SC_OK, responseBody.length);
                exchange.getResponseBody().write(responseBody);
                return;
            }
        }
        fail("did not send response");
    });
    try (Ec2DiscoveryPlugin plugin = new Ec2DiscoveryPlugin(buildSettings(accessKey))) {
        final SeedHostsProvider seedHostsProvider = plugin.getSeedHostProviders(transportService, networkService).get("ec2").get();
        final SeedHostsResolver resolver = new SeedHostsResolver("test", Settings.EMPTY, transportService, seedHostsProvider);
        resolver.start();
        final List<TransportAddress> addressList = seedHostsProvider.getSeedAddresses(null);
        assertThat(addressList, Matchers.hasSize(1));
        assertThat(addressList.get(0).toString(), is(hosts.get(0)));
        assertThat(failedRequests, aMapWithSize(1));
        assertThat(failedRequests.values().iterator().next(), is(maxRetries));
    }
}

Example 45 with TransportService

use of org.opensearch.transport.TransportService in project OpenSearch by opensearch-project.

the class AbstractAzureComputeServiceTestCase method registerAzureNode.

/**
 * Register an existing node as a Azure node, exposing its address and details htrough
 *
 * @param nodeName the name of the node
 */
protected void registerAzureNode(final String nodeName) {
    TransportService transportService = internalCluster().getInstance(TransportService.class, nodeName);
    assertNotNull(transportService);
    DiscoveryNode discoveryNode = transportService.getLocalNode();
    assertNotNull(discoveryNode);
    if (nodes.put(discoveryNode.getName(), discoveryNode) != null) {
        throw new IllegalArgumentException("Node [" + discoveryNode.getName() + "] cannot be registered twice in Azure");
    }
}