Examples with AtomicArray org.opensearch.common.util.concurrent.AtomicArray used on opensource projects

Example 1 with AtomicArray

use of org.opensearch.common.util.concurrent.AtomicArray in project OpenSearch by opensearch-project.

the class TransportBulkAction method doInternalExecute.

protected void doInternalExecute(Task task, BulkRequest bulkRequest, String executorName, ActionListener<BulkResponse> listener) {
    final long startTime = relativeTime();
    final AtomicArray<BulkItemResponse> responses = new AtomicArray<>(bulkRequest.requests.size());
    boolean hasIndexRequestsWithPipelines = false;
    final Metadata metadata = clusterService.state().getMetadata();
    final Version minNodeVersion = clusterService.state().getNodes().getMinNodeVersion();
    for (DocWriteRequest<?> actionRequest : bulkRequest.requests) {
        IndexRequest indexRequest = getIndexWriteRequest(actionRequest);
        if (indexRequest != null) {
            // Each index request needs to be evaluated, because this method also modifies the IndexRequest
            boolean indexRequestHasPipeline = IngestService.resolvePipelines(actionRequest, indexRequest, metadata);
            hasIndexRequestsWithPipelines |= indexRequestHasPipeline;
        }
        if (actionRequest instanceof IndexRequest) {
            IndexRequest ir = (IndexRequest) actionRequest;
            ir.checkAutoIdWithOpTypeCreateSupportedByVersion(minNodeVersion);
            if (ir.getAutoGeneratedTimestamp() != IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP) {
                throw new IllegalArgumentException("autoGeneratedTimestamp should not be set externally");
            }
        }
    }
    if (hasIndexRequestsWithPipelines) {
        // this path is never taken.
        try {
            if (Assertions.ENABLED) {
                final boolean arePipelinesResolved = bulkRequest.requests().stream().map(TransportBulkAction::getIndexWriteRequest).filter(Objects::nonNull).allMatch(IndexRequest::isPipelineResolved);
                assert arePipelinesResolved : bulkRequest;
            }
            if (clusterService.localNode().isIngestNode()) {
                processBulkIndexIngestRequest(task, bulkRequest, executorName, listener);
            } else {
                ingestForwarder.forwardIngestRequest(BulkAction.INSTANCE, bulkRequest, listener);
            }
        } catch (Exception e) {
            listener.onFailure(e);
        }
        return;
    }
    final boolean includesSystem = includesSystem(bulkRequest, clusterService.state().metadata().getIndicesLookup(), systemIndices);
    if (includesSystem || needToCheck()) {
        // Attempt to create all the indices that we're going to need during the bulk before we start.
        // Step 1: collect all the indices in the request
        final Map<String, Boolean> indices = bulkRequest.requests.stream().filter(request -> request.opType() != DocWriteRequest.OpType.DELETE || request.versionType() == VersionType.EXTERNAL || request.versionType() == VersionType.EXTERNAL_GTE).collect(Collectors.toMap(DocWriteRequest::index, DocWriteRequest::isRequireAlias, (v1, v2) -> v1 || v2));
        /* Step 2: filter that to indices that don't exist and we can create. At the same time build a map of indices we can't create
             * that we'll use when we try to run the requests. */
        final Map<String, IndexNotFoundException> indicesThatCannotBeCreated = new HashMap<>();
        Set<String> autoCreateIndices = new HashSet<>();
        ClusterState state = clusterService.state();
        for (Map.Entry<String, Boolean> indexAndFlag : indices.entrySet()) {
            boolean shouldAutoCreate;
            final String index = indexAndFlag.getKey();
            try {
                shouldAutoCreate = shouldAutoCreate(index, state);
            } catch (IndexNotFoundException e) {
                shouldAutoCreate = false;
                indicesThatCannotBeCreated.put(index, e);
            }
            // We should only auto create if we are not requiring it to be an alias
            if (shouldAutoCreate && (indexAndFlag.getValue() == false)) {
                autoCreateIndices.add(index);
            }
        }
        // Step 3: create all the indices that are missing, if there are any missing. start the bulk after all the creates come back.
        if (autoCreateIndices.isEmpty()) {
            executeBulk(task, bulkRequest, startTime, listener, responses, indicesThatCannotBeCreated);
        } else {
            final AtomicInteger counter = new AtomicInteger(autoCreateIndices.size());
            for (String index : autoCreateIndices) {
                createIndex(index, bulkRequest.timeout(), minNodeVersion, new ActionListener<CreateIndexResponse>() {

                    @Override
                    public void onResponse(CreateIndexResponse result) {
                        if (counter.decrementAndGet() == 0) {
                            threadPool.executor(executorName).execute(new ActionRunnable<BulkResponse>(listener) {

                                @Override
                                protected void doRun() {
                                    executeBulk(task, bulkRequest, startTime, listener, responses, indicesThatCannotBeCreated);
                                }
                            });
                        }
                    }

                    @Override
                    public void onFailure(Exception e) {
                        if (!(ExceptionsHelper.unwrapCause(e) instanceof ResourceAlreadyExistsException)) {
                            // fail all requests involving this index, if create didn't work
                            for (int i = 0; i < bulkRequest.requests.size(); i++) {
                                DocWriteRequest<?> request = bulkRequest.requests.get(i);
                                if (request != null && setResponseFailureIfIndexMatches(responses, i, request, index, e)) {
                                    bulkRequest.requests.set(i, null);
                                }
                            }
                        }
                        if (counter.decrementAndGet() == 0) {
                            final ActionListener<BulkResponse> wrappedListener = ActionListener.wrap(listener::onResponse, inner -> {
                                inner.addSuppressed(e);
                                listener.onFailure(inner);
                            });
                            threadPool.executor(executorName).execute(new ActionRunnable<BulkResponse>(wrappedListener) {

                                @Override
                                protected void doRun() {
                                    executeBulk(task, bulkRequest, startTime, wrappedListener, responses, indicesThatCannotBeCreated);
                                }

                                @Override
                                public void onRejection(Exception rejectedException) {
                                    rejectedException.addSuppressed(e);
                                    super.onRejection(rejectedException);
                                }
                            });
                        }
                    }
                });
            }
        }
    } else {
        executeBulk(task, bulkRequest, startTime, listener, responses, emptyMap());
    }
}

Example 2 with AtomicArray

use of org.opensearch.common.util.concurrent.AtomicArray in project OpenSearch by opensearch-project.

the class TransportMultiTermVectorsAction method doExecute.

@Override
protected void doExecute(Task task, final MultiTermVectorsRequest request, final ActionListener<MultiTermVectorsResponse> listener) {
    ClusterState clusterState = clusterService.state();
    clusterState.blocks().globalBlockedRaiseException(ClusterBlockLevel.READ);
    final AtomicArray<MultiTermVectorsItemResponse> responses = new AtomicArray<>(request.requests.size());
    Map<ShardId, MultiTermVectorsShardRequest> shardRequests = new HashMap<>();
    for (int i = 0; i < request.requests.size(); i++) {
        TermVectorsRequest termVectorsRequest = request.requests.get(i);
        termVectorsRequest.routing(clusterState.metadata().resolveIndexRouting(termVectorsRequest.routing(), termVectorsRequest.index()));
        if (!clusterState.metadata().hasConcreteIndex(termVectorsRequest.index())) {
            responses.set(i, new MultiTermVectorsItemResponse(null, new MultiTermVectorsResponse.Failure(termVectorsRequest.index(), termVectorsRequest.id(), new IndexNotFoundException(termVectorsRequest.index()))));
            continue;
        }
        String concreteSingleIndex = indexNameExpressionResolver.concreteSingleIndex(clusterState, termVectorsRequest).getName();
        if (termVectorsRequest.routing() == null && clusterState.getMetadata().routingRequired(concreteSingleIndex)) {
            responses.set(i, new MultiTermVectorsItemResponse(null, new MultiTermVectorsResponse.Failure(concreteSingleIndex, termVectorsRequest.id(), new RoutingMissingException(concreteSingleIndex, termVectorsRequest.id()))));
            continue;
        }
        ShardId shardId = clusterService.operationRouting().shardId(clusterState, concreteSingleIndex, termVectorsRequest.id(), termVectorsRequest.routing());
        MultiTermVectorsShardRequest shardRequest = shardRequests.get(shardId);
        if (shardRequest == null) {
            shardRequest = new MultiTermVectorsShardRequest(shardId.getIndexName(), shardId.id());
            shardRequest.preference(request.preference);
            shardRequests.put(shardId, shardRequest);
        }
        shardRequest.add(i, termVectorsRequest);
    }
    if (shardRequests.size() == 0) {
        // only failures..
        listener.onResponse(new MultiTermVectorsResponse(responses.toArray(new MultiTermVectorsItemResponse[responses.length()])));
    }
    executeShardAction(listener, responses, shardRequests);
}

Example 3 with AtomicArray

use of org.opensearch.common.util.concurrent.AtomicArray in project OpenSearch by opensearch-project.

the class ClearScrollControllerTests method testClearScrollIdsWithFailure.

public void testClearScrollIdsWithFailure() throws IOException, InterruptedException {
    DiscoveryNode node1 = new DiscoveryNode("node_1", buildNewFakeTransportAddress(), Version.CURRENT);
    DiscoveryNode node2 = new DiscoveryNode("node_2", buildNewFakeTransportAddress(), Version.CURRENT);
    DiscoveryNode node3 = new DiscoveryNode("node_3", buildNewFakeTransportAddress(), Version.CURRENT);
    AtomicArray<SearchPhaseResult> array = new AtomicArray<>(3);
    SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult1 = new SearchAsyncActionTests.TestSearchPhaseResult(new ShardSearchContextId(UUIDs.randomBase64UUID(), 1), node1);
    testSearchPhaseResult1.setSearchShardTarget(new SearchShardTarget("node_1", new ShardId("idx", "uuid1", 2), null, null));
    SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult2 = new SearchAsyncActionTests.TestSearchPhaseResult(new ShardSearchContextId(UUIDs.randomBase64UUID(), 12), node2);
    testSearchPhaseResult2.setSearchShardTarget(new SearchShardTarget("node_2", new ShardId("idy", "uuid2", 42), null, null));
    SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult3 = new SearchAsyncActionTests.TestSearchPhaseResult(new ShardSearchContextId(UUIDs.randomBase64UUID(), 42), node3);
    testSearchPhaseResult3.setSearchShardTarget(new SearchShardTarget("node_3", new ShardId("idy", "uuid2", 43), null, null));
    array.setOnce(0, testSearchPhaseResult1);
    array.setOnce(1, testSearchPhaseResult2);
    array.setOnce(2, testSearchPhaseResult3);
    AtomicInteger numFreed = new AtomicInteger(0);
    AtomicInteger numFailures = new AtomicInteger(0);
    AtomicInteger numConnectionFailures = new AtomicInteger(0);
    String scrollId = TransportSearchHelper.buildScrollId(array, VersionUtils.randomVersion(random()));
    DiscoveryNodes nodes = DiscoveryNodes.builder().add(node1).add(node2).add(node3).build();
    CountDownLatch latch = new CountDownLatch(1);
    ActionListener<ClearScrollResponse> listener = new LatchedActionListener<>(new ActionListener<ClearScrollResponse>() {

        @Override
        public void onResponse(ClearScrollResponse clearScrollResponse) {
            assertEquals(numFreed.get(), clearScrollResponse.getNumFreed());
            if (numFailures.get() > 0) {
                assertFalse(clearScrollResponse.isSucceeded());
            } else {
                assertTrue(clearScrollResponse.isSucceeded());
            }
        }

        @Override
        public void onFailure(Exception e) {
            throw new AssertionError(e);
        }
    }, latch);
    List<DiscoveryNode> nodesInvoked = new CopyOnWriteArrayList<>();
    SearchTransportService searchTransportService = new SearchTransportService(null, null) {

        @Override
        public void sendFreeContext(Transport.Connection connection, ShardSearchContextId contextId, ActionListener<SearchFreeContextResponse> listener) {
            nodesInvoked.add(connection.getNode());
            boolean freed = randomBoolean();
            boolean fail = randomBoolean();
            Thread t = new Thread(() -> {
                if (fail) {
                    numFailures.incrementAndGet();
                    listener.onFailure(new IllegalArgumentException("boom"));
                } else {
                    if (freed) {
                        numFreed.incrementAndGet();
                    }
                    listener.onResponse(new SearchFreeContextResponse(freed));
                }
            });
            t.start();
        }

        @Override
        public Transport.Connection getConnection(String clusterAlias, DiscoveryNode node) {
            if (randomBoolean()) {
                numFailures.incrementAndGet();
                numConnectionFailures.incrementAndGet();
                throw new NodeNotConnectedException(node, "boom");
            }
            return new SearchAsyncActionTests.MockConnection(node);
        }
    };
    ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
    clearScrollRequest.scrollIds(Arrays.asList(scrollId));
    ClearScrollController controller = new ClearScrollController(clearScrollRequest, listener, nodes, logger, searchTransportService);
    controller.run();
    latch.await();
    assertEquals(3 - numConnectionFailures.get(), nodesInvoked.size());
}

Example 4 with AtomicArray

use of org.opensearch.common.util.concurrent.AtomicArray in project OpenSearch by opensearch-project.

the class ClearScrollControllerTests method testClearScrollIds.

public void testClearScrollIds() throws IOException, InterruptedException {
    DiscoveryNode node1 = new DiscoveryNode("node_1", buildNewFakeTransportAddress(), Version.CURRENT);
    DiscoveryNode node2 = new DiscoveryNode("node_2", buildNewFakeTransportAddress(), Version.CURRENT);
    DiscoveryNode node3 = new DiscoveryNode("node_3", buildNewFakeTransportAddress(), Version.CURRENT);
    AtomicArray<SearchPhaseResult> array = new AtomicArray<>(3);
    SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult1 = new SearchAsyncActionTests.TestSearchPhaseResult(new ShardSearchContextId(UUIDs.randomBase64UUID(), 1), node1);
    testSearchPhaseResult1.setSearchShardTarget(new SearchShardTarget("node_1", new ShardId("idx", "uuid1", 2), null, null));
    SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult2 = new SearchAsyncActionTests.TestSearchPhaseResult(new ShardSearchContextId(UUIDs.randomBase64UUID(), 12), node2);
    testSearchPhaseResult2.setSearchShardTarget(new SearchShardTarget("node_2", new ShardId("idy", "uuid2", 42), null, null));
    SearchAsyncActionTests.TestSearchPhaseResult testSearchPhaseResult3 = new SearchAsyncActionTests.TestSearchPhaseResult(new ShardSearchContextId(UUIDs.randomBase64UUID(), 42), node3);
    testSearchPhaseResult3.setSearchShardTarget(new SearchShardTarget("node_3", new ShardId("idy", "uuid2", 43), null, null));
    array.setOnce(0, testSearchPhaseResult1);
    array.setOnce(1, testSearchPhaseResult2);
    array.setOnce(2, testSearchPhaseResult3);
    AtomicInteger numFreed = new AtomicInteger(0);
    String scrollId = TransportSearchHelper.buildScrollId(array, VersionUtils.randomVersion(random()));
    DiscoveryNodes nodes = DiscoveryNodes.builder().add(node1).add(node2).add(node3).build();
    CountDownLatch latch = new CountDownLatch(1);
    ActionListener<ClearScrollResponse> listener = new LatchedActionListener<>(new ActionListener<ClearScrollResponse>() {

        @Override
        public void onResponse(ClearScrollResponse clearScrollResponse) {
            assertEquals(numFreed.get(), clearScrollResponse.getNumFreed());
            assertTrue(clearScrollResponse.isSucceeded());
        }

        @Override
        public void onFailure(Exception e) {
            throw new AssertionError(e);
        }
    }, latch);
    List<DiscoveryNode> nodesInvoked = new CopyOnWriteArrayList<>();
    SearchTransportService searchTransportService = new SearchTransportService(null, null) {

        @Override
        public void sendFreeContext(Transport.Connection connection, ShardSearchContextId contextId, ActionListener<SearchFreeContextResponse> listener) {
            nodesInvoked.add(connection.getNode());
            boolean freed = randomBoolean();
            if (freed) {
                numFreed.incrementAndGet();
            }
            Thread t = new Thread(() -> listener.onResponse(new SearchFreeContextResponse(freed)));
            t.start();
        }

        @Override
        public Transport.Connection getConnection(String clusterAlias, DiscoveryNode node) {
            return new SearchAsyncActionTests.MockConnection(node);
        }
    };
    ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
    clearScrollRequest.scrollIds(Arrays.asList(scrollId));
    ClearScrollController controller = new ClearScrollController(clearScrollRequest, listener, nodes, logger, searchTransportService);
    controller.run();
    latch.await();
    assertEquals(3, nodesInvoked.size());
    Collections.sort(nodesInvoked, Comparator.comparing(DiscoveryNode::getId));
    assertEquals(nodesInvoked, Arrays.asList(node1, node2, node3));
}

Example 5 with AtomicArray

use of org.opensearch.common.util.concurrent.AtomicArray in project OpenSearch by opensearch-project.

the class CountedCollectorTests method testCollect.

public void testCollect() throws InterruptedException {
    ArraySearchPhaseResults<SearchPhaseResult> consumer = new ArraySearchPhaseResults<>(randomIntBetween(1, 100));
    List<Integer> state = new ArrayList<>();
    int numResultsExpected = randomIntBetween(1, consumer.getAtomicArray().length());
    MockSearchPhaseContext context = new MockSearchPhaseContext(consumer.getAtomicArray().length());
    CountDownLatch latch = new CountDownLatch(1);
    boolean maybeFork = randomBoolean();
    Executor executor = (runnable) -> {
        if (randomBoolean() && maybeFork) {
            new Thread(runnable).start();
        } else {
            runnable.run();
        }
    };
    CountedCollector<SearchPhaseResult> collector = new CountedCollector<>(consumer, numResultsExpected, latch::countDown, context);
    for (int i = 0; i < numResultsExpected; i++) {
        int shardID = i;
        switch(randomIntBetween(0, 2)) {
            case 0:
                state.add(0);
                executor.execute(() -> collector.countDown());
                break;
            case 1:
                state.add(1);
                executor.execute(() -> {
                    DfsSearchResult dfsSearchResult = new DfsSearchResult(new ShardSearchContextId(UUIDs.randomBase64UUID(), shardID), null, null);
                    dfsSearchResult.setShardIndex(shardID);
                    dfsSearchResult.setSearchShardTarget(new SearchShardTarget("foo", new ShardId("bar", "baz", shardID), null, OriginalIndices.NONE));
                    collector.onResult(dfsSearchResult);
                });
                break;
            case 2:
                state.add(2);
                executor.execute(() -> collector.onFailure(shardID, new SearchShardTarget("foo", new ShardId("bar", "baz", shardID), null, OriginalIndices.NONE), new RuntimeException("boom")));
                break;
            default:
                fail("unknown state");
        }
    }
    latch.await();
    assertEquals(numResultsExpected, state.size());
    AtomicArray<SearchPhaseResult> results = consumer.getAtomicArray();
    for (int i = 0; i < numResultsExpected; i++) {
        switch(state.get(i)) {
            case 0:
                assertNull(results.get(i));
                break;
            case 1:
                assertNotNull(results.get(i));
                assertEquals(i, results.get(i).getContextId().getId());
                break;
            case 2:
                final int shardId = i;
                assertEquals(1, context.failures.stream().filter(f -> f.shardId() == shardId).count());
                break;
            default:
                fail("unknown state");
        }
    }
    for (int i = numResultsExpected; i < results.length(); i++) {
        assertNull("index: " + i, results.get(i));
    }
}