Search in sources :

Example 1 with ActionRunnable

use of org.opensearch.action.ActionRunnable in project OpenSearch by opensearch-project.

the class TransportBulkAction method doInternalExecute.

protected void doInternalExecute(Task task, BulkRequest bulkRequest, String executorName, ActionListener<BulkResponse> listener) {
    final long startTime = relativeTime();
    final AtomicArray<BulkItemResponse> responses = new AtomicArray<>(bulkRequest.requests.size());
    boolean hasIndexRequestsWithPipelines = false;
    final Metadata metadata = clusterService.state().getMetadata();
    final Version minNodeVersion = clusterService.state().getNodes().getMinNodeVersion();
    for (DocWriteRequest<?> actionRequest : bulkRequest.requests) {
        IndexRequest indexRequest = getIndexWriteRequest(actionRequest);
        if (indexRequest != null) {
            // Each index request needs to be evaluated, because this method also modifies the IndexRequest
            boolean indexRequestHasPipeline = IngestService.resolvePipelines(actionRequest, indexRequest, metadata);
            hasIndexRequestsWithPipelines |= indexRequestHasPipeline;
        }
        if (actionRequest instanceof IndexRequest) {
            IndexRequest ir = (IndexRequest) actionRequest;
            ir.checkAutoIdWithOpTypeCreateSupportedByVersion(minNodeVersion);
            if (ir.getAutoGeneratedTimestamp() != IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP) {
                throw new IllegalArgumentException("autoGeneratedTimestamp should not be set externally");
            }
        }
    }
    if (hasIndexRequestsWithPipelines) {
        // this path is never taken.
        try {
            if (Assertions.ENABLED) {
                final boolean arePipelinesResolved = bulkRequest.requests().stream().map(TransportBulkAction::getIndexWriteRequest).filter(Objects::nonNull).allMatch(IndexRequest::isPipelineResolved);
                assert arePipelinesResolved : bulkRequest;
            }
            if (clusterService.localNode().isIngestNode()) {
                processBulkIndexIngestRequest(task, bulkRequest, executorName, listener);
            } else {
                ingestForwarder.forwardIngestRequest(BulkAction.INSTANCE, bulkRequest, listener);
            }
        } catch (Exception e) {
            listener.onFailure(e);
        }
        return;
    }
    final boolean includesSystem = includesSystem(bulkRequest, clusterService.state().metadata().getIndicesLookup(), systemIndices);
    if (includesSystem || needToCheck()) {
        // Attempt to create all the indices that we're going to need during the bulk before we start.
        // Step 1: collect all the indices in the request
        final Map<String, Boolean> indices = bulkRequest.requests.stream().filter(request -> request.opType() != DocWriteRequest.OpType.DELETE || request.versionType() == VersionType.EXTERNAL || request.versionType() == VersionType.EXTERNAL_GTE).collect(Collectors.toMap(DocWriteRequest::index, DocWriteRequest::isRequireAlias, (v1, v2) -> v1 || v2));
        /* Step 2: filter that to indices that don't exist and we can create. At the same time build a map of indices we can't create
             * that we'll use when we try to run the requests. */
        final Map<String, IndexNotFoundException> indicesThatCannotBeCreated = new HashMap<>();
        Set<String> autoCreateIndices = new HashSet<>();
        ClusterState state = clusterService.state();
        for (Map.Entry<String, Boolean> indexAndFlag : indices.entrySet()) {
            boolean shouldAutoCreate;
            final String index = indexAndFlag.getKey();
            try {
                shouldAutoCreate = shouldAutoCreate(index, state);
            } catch (IndexNotFoundException e) {
                shouldAutoCreate = false;
                indicesThatCannotBeCreated.put(index, e);
            }
            // We should only auto create if we are not requiring it to be an alias
            if (shouldAutoCreate && (indexAndFlag.getValue() == false)) {
                autoCreateIndices.add(index);
            }
        }
        // Step 3: create all the indices that are missing, if there are any missing. start the bulk after all the creates come back.
        if (autoCreateIndices.isEmpty()) {
            executeBulk(task, bulkRequest, startTime, listener, responses, indicesThatCannotBeCreated);
        } else {
            final AtomicInteger counter = new AtomicInteger(autoCreateIndices.size());
            for (String index : autoCreateIndices) {
                createIndex(index, bulkRequest.timeout(), minNodeVersion, new ActionListener<CreateIndexResponse>() {

                    @Override
                    public void onResponse(CreateIndexResponse result) {
                        if (counter.decrementAndGet() == 0) {
                            threadPool.executor(executorName).execute(new ActionRunnable<BulkResponse>(listener) {

                                @Override
                                protected void doRun() {
                                    executeBulk(task, bulkRequest, startTime, listener, responses, indicesThatCannotBeCreated);
                                }
                            });
                        }
                    }

                    @Override
                    public void onFailure(Exception e) {
                        if (!(ExceptionsHelper.unwrapCause(e) instanceof ResourceAlreadyExistsException)) {
                            // fail all requests involving this index, if create didn't work
                            for (int i = 0; i < bulkRequest.requests.size(); i++) {
                                DocWriteRequest<?> request = bulkRequest.requests.get(i);
                                if (request != null && setResponseFailureIfIndexMatches(responses, i, request, index, e)) {
                                    bulkRequest.requests.set(i, null);
                                }
                            }
                        }
                        if (counter.decrementAndGet() == 0) {
                            final ActionListener<BulkResponse> wrappedListener = ActionListener.wrap(listener::onResponse, inner -> {
                                inner.addSuppressed(e);
                                listener.onFailure(inner);
                            });
                            threadPool.executor(executorName).execute(new ActionRunnable<BulkResponse>(wrappedListener) {

                                @Override
                                protected void doRun() {
                                    executeBulk(task, bulkRequest, startTime, wrappedListener, responses, indicesThatCannotBeCreated);
                                }

                                @Override
                                public void onRejection(Exception rejectedException) {
                                    rejectedException.addSuppressed(e);
                                    super.onRejection(rejectedException);
                                }
                            });
                        }
                    }
                });
            }
        }
    } else {
        executeBulk(task, bulkRequest, startTime, listener, responses, emptyMap());
    }
}
Also used : SequenceNumbers(org.opensearch.index.seqno.SequenceNumbers) IndexAbstraction(org.opensearch.cluster.metadata.IndexAbstraction) Metadata(org.opensearch.cluster.metadata.Metadata) LongSupplier(java.util.function.LongSupplier) DataStream(org.opensearch.cluster.metadata.DataStream) Version(org.opensearch.Version) TransportUpdateAction(org.opensearch.action.update.TransportUpdateAction) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Locale(java.util.Locale) Assertions(org.opensearch.Assertions) Map(java.util.Map) NodeClosedException(org.opensearch.node.NodeClosedException) AutoCreateAction(org.opensearch.action.admin.indices.create.AutoCreateAction) Inject(org.opensearch.common.inject.Inject) ActionListener(org.opensearch.action.ActionListener) AtomicIntegerArray(java.util.concurrent.atomic.AtomicIntegerArray) CreateIndexRequest(org.opensearch.action.admin.indices.create.CreateIndexRequest) TimeValue(org.opensearch.common.unit.TimeValue) NodeClient(org.opensearch.client.node.NodeClient) Index(org.opensearch.index.Index) IndexingPressureService(org.opensearch.index.IndexingPressureService) OpenSearchParseException(org.opensearch.OpenSearchParseException) ExceptionsHelper(org.opensearch.ExceptionsHelper) ClusterBlockException(org.opensearch.cluster.block.ClusterBlockException) Set(java.util.Set) Task(org.opensearch.tasks.Task) TransportService(org.opensearch.transport.TransportService) Collectors(java.util.stream.Collectors) Objects(java.util.Objects) ActionFilters(org.opensearch.action.support.ActionFilters) VersionType(org.opensearch.index.VersionType) List(java.util.List) Logger(org.apache.logging.log4j.Logger) SparseFixedBitSet(org.apache.lucene.util.SparseFixedBitSet) EXCLUDED_DATA_STREAMS_KEY(org.opensearch.cluster.metadata.IndexNameExpressionResolver.EXCLUDED_DATA_STREAMS_KEY) ResourceAlreadyExistsException(org.opensearch.ResourceAlreadyExistsException) DocWriteResponse(org.opensearch.action.DocWriteResponse) UpdateRequest(org.opensearch.action.update.UpdateRequest) SortedMap(java.util.SortedMap) IndexNameExpressionResolver(org.opensearch.cluster.metadata.IndexNameExpressionResolver) Names(org.opensearch.threadpool.ThreadPool.Names) MappingMetadata(org.opensearch.cluster.metadata.MappingMetadata) HandledTransportAction(org.opensearch.action.support.HandledTransportAction) IndexMetadata(org.opensearch.cluster.metadata.IndexMetadata) ActionRunnable(org.opensearch.action.ActionRunnable) UpdateResponse(org.opensearch.action.update.UpdateResponse) ThreadPool(org.opensearch.threadpool.ThreadPool) RoutingMissingException(org.opensearch.action.RoutingMissingException) DocWriteRequest(org.opensearch.action.DocWriteRequest) HashMap(java.util.HashMap) Releasable(org.opensearch.common.lease.Releasable) ArrayList(java.util.ArrayList) AutoCreateIndex(org.opensearch.action.support.AutoCreateIndex) HashSet(java.util.HashSet) ClusterState(org.opensearch.cluster.ClusterState) UNASSIGNED_SEQ_NO(org.opensearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO) LegacyESVersion(org.opensearch.LegacyESVersion) IndexClosedException(org.opensearch.indices.IndexClosedException) ClusterStateObserver(org.opensearch.cluster.ClusterStateObserver) Collections.emptyMap(java.util.Collections.emptyMap) IngestService(org.opensearch.ingest.IngestService) Iterator(java.util.Iterator) IngestActionForwarder(org.opensearch.action.ingest.IngestActionForwarder) ClusterBlockLevel(org.opensearch.cluster.block.ClusterBlockLevel) IndexNotFoundException(org.opensearch.index.IndexNotFoundException) CreateIndexResponse(org.opensearch.action.admin.indices.create.CreateIndexResponse) UNASSIGNED_PRIMARY_TERM(org.opensearch.index.seqno.SequenceNumbers.UNASSIGNED_PRIMARY_TERM) ShardId(org.opensearch.index.shard.ShardId) TimeUnit(java.util.concurrent.TimeUnit) SystemIndices(org.opensearch.indices.SystemIndices) AtomicArray(org.opensearch.common.util.concurrent.AtomicArray) ClusterService(org.opensearch.cluster.service.ClusterService) IndexRequest(org.opensearch.action.index.IndexRequest) LogManager(org.apache.logging.log4j.LogManager) AtomicArray(org.opensearch.common.util.concurrent.AtomicArray) ActionRunnable(org.opensearch.action.ActionRunnable) HashMap(java.util.HashMap) Metadata(org.opensearch.cluster.metadata.Metadata) MappingMetadata(org.opensearch.cluster.metadata.MappingMetadata) IndexMetadata(org.opensearch.cluster.metadata.IndexMetadata) CreateIndexRequest(org.opensearch.action.admin.indices.create.CreateIndexRequest) IndexRequest(org.opensearch.action.index.IndexRequest) Version(org.opensearch.Version) LegacyESVersion(org.opensearch.LegacyESVersion) CreateIndexResponse(org.opensearch.action.admin.indices.create.CreateIndexResponse) HashSet(java.util.HashSet) ClusterState(org.opensearch.cluster.ClusterState) ResourceAlreadyExistsException(org.opensearch.ResourceAlreadyExistsException) NodeClosedException(org.opensearch.node.NodeClosedException) OpenSearchParseException(org.opensearch.OpenSearchParseException) ClusterBlockException(org.opensearch.cluster.block.ClusterBlockException) ResourceAlreadyExistsException(org.opensearch.ResourceAlreadyExistsException) RoutingMissingException(org.opensearch.action.RoutingMissingException) IndexClosedException(org.opensearch.indices.IndexClosedException) IndexNotFoundException(org.opensearch.index.IndexNotFoundException) ActionListener(org.opensearch.action.ActionListener) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) IndexNotFoundException(org.opensearch.index.IndexNotFoundException) DocWriteRequest(org.opensearch.action.DocWriteRequest) Map(java.util.Map) SortedMap(java.util.SortedMap) HashMap(java.util.HashMap) Collections.emptyMap(java.util.Collections.emptyMap)

Example 2 with ActionRunnable

use of org.opensearch.action.ActionRunnable in project OpenSearch by opensearch-project.

the class TransportShardBulkAction method performOnPrimary.

public static void performOnPrimary(BulkShardRequest request, IndexShard primary, UpdateHelper updateHelper, LongSupplier nowInMillisSupplier, MappingUpdatePerformer mappingUpdater, Consumer<ActionListener<Void>> waitForMappingUpdate, ActionListener<PrimaryResult<BulkShardRequest, BulkShardResponse>> listener, ThreadPool threadPool, String executorName) {
    new ActionRunnable<PrimaryResult<BulkShardRequest, BulkShardResponse>>(listener) {

        private final Executor executor = threadPool.executor(executorName);

        private final BulkPrimaryExecutionContext context = new BulkPrimaryExecutionContext(request, primary);

        @Override
        protected void doRun() throws Exception {
            while (context.hasMoreOperationsToExecute()) {
                if (executeBulkItemRequest(context, updateHelper, nowInMillisSupplier, mappingUpdater, waitForMappingUpdate, ActionListener.wrap(v -> executor.execute(this), this::onRejection)) == false) {
                    // so we just break out here.
                    return;
                }
                // either completed and moved to next or reset
                assert context.isInitial();
            }
            // We're done, there's no more operations to execute so we resolve the wrapped listener
            finishRequest();
        }

        @Override
        public void onRejection(Exception e) {
            // We must finish the outstanding request. Finishing the outstanding request can include
            // refreshing and fsyncing. Therefore, we must force execution on the WRITE thread.
            executor.execute(new ActionRunnable<PrimaryResult<BulkShardRequest, BulkShardResponse>>(listener) {

                @Override
                protected void doRun() {
                    // Fail all operations after a bulk rejection hit an action that waited for a mapping update and finish the request
                    while (context.hasMoreOperationsToExecute()) {
                        context.setRequestToExecute(context.getCurrent());
                        final DocWriteRequest<?> docWriteRequest = context.getRequestToExecute();
                        onComplete(exceptionToResult(e, primary, docWriteRequest.opType() == DocWriteRequest.OpType.DELETE, docWriteRequest.version()), context, null);
                    }
                    finishRequest();
                }

                @Override
                public boolean isForceExecution() {
                    return true;
                }
            });
        }

        private void finishRequest() {
            ActionListener.completeWith(listener, () -> new WritePrimaryResult<>(context.getBulkShardRequest(), context.buildShardResponse(), context.getLocationToSync(), null, context.getPrimary(), logger));
        }
    }.run();
}
Also used : SequenceNumbers(org.opensearch.index.seqno.SequenceNumbers) LongSupplier(java.util.function.LongSupplier) IndexResponse(org.opensearch.action.index.IndexResponse) ToXContent(org.opensearch.common.xcontent.ToXContent) MapperService(org.opensearch.index.mapper.MapperService) Map(java.util.Map) NodeClosedException(org.opensearch.node.NodeClosedException) Inject(org.opensearch.common.inject.Inject) ActionListener(org.opensearch.action.ActionListener) ShardStateAction(org.opensearch.cluster.action.shard.ShardStateAction) DeleteRequest(org.opensearch.action.delete.DeleteRequest) MapperException(org.opensearch.index.mapper.MapperException) TimeValue(org.opensearch.common.unit.TimeValue) IndexingPressureService(org.opensearch.index.IndexingPressureService) GetResult(org.opensearch.index.get.GetResult) IndicesService(org.opensearch.indices.IndicesService) TransportRequestOptions(org.opensearch.transport.TransportRequestOptions) ExceptionsHelper(org.opensearch.ExceptionsHelper) Settings(org.opensearch.common.settings.Settings) TransportService(org.opensearch.transport.TransportService) Tuple(org.opensearch.common.collect.Tuple) Engine(org.opensearch.index.engine.Engine) ActionFilters(org.opensearch.action.support.ActionFilters) Logger(org.apache.logging.log4j.Logger) DocWriteResponse(org.opensearch.action.DocWriteResponse) UpdateRequest(org.opensearch.action.update.UpdateRequest) XContentType(org.opensearch.common.xcontent.XContentType) Names(org.opensearch.threadpool.ThreadPool.Names) MappingMetadata(org.opensearch.cluster.metadata.MappingMetadata) MessageSupplier(org.apache.logging.log4j.util.MessageSupplier) BytesReference(org.opensearch.common.bytes.BytesReference) IndexMetadata(org.opensearch.cluster.metadata.IndexMetadata) ActionRunnable(org.opensearch.action.ActionRunnable) UpdateResponse(org.opensearch.action.update.UpdateResponse) ThreadPool(org.opensearch.threadpool.ThreadPool) CompressedXContent(org.opensearch.common.compress.CompressedXContent) DocWriteRequest(org.opensearch.action.DocWriteRequest) ParameterizedMessage(org.apache.logging.log4j.message.ParameterizedMessage) Function(java.util.function.Function) MappingUpdatedAction(org.opensearch.cluster.action.index.MappingUpdatedAction) SourceToParse(org.opensearch.index.mapper.SourceToParse) ClusterState(org.opensearch.cluster.ClusterState) IndexShard(org.opensearch.index.shard.IndexShard) DeleteResponse(org.opensearch.action.delete.DeleteResponse) Translog(org.opensearch.index.translog.Translog) TransportWriteAction(org.opensearch.action.support.replication.TransportWriteAction) ClusterStateObserver(org.opensearch.cluster.ClusterStateObserver) StreamInput(org.opensearch.common.io.stream.StreamInput) UpdateHelper(org.opensearch.action.update.UpdateHelper) Executor(java.util.concurrent.Executor) VersionConflictEngineException(org.opensearch.index.engine.VersionConflictEngineException) IOException(java.io.IOException) TransportReplicationAction(org.opensearch.action.support.replication.TransportReplicationAction) XContentHelper(org.opensearch.common.xcontent.XContentHelper) ShardId(org.opensearch.index.shard.ShardId) Consumer(java.util.function.Consumer) SystemIndices(org.opensearch.indices.SystemIndices) ClusterService(org.opensearch.cluster.service.ClusterService) IndexRequest(org.opensearch.action.index.IndexRequest) LogManager(org.apache.logging.log4j.LogManager) ActionRunnable(org.opensearch.action.ActionRunnable) Executor(java.util.concurrent.Executor) NodeClosedException(org.opensearch.node.NodeClosedException) MapperException(org.opensearch.index.mapper.MapperException) VersionConflictEngineException(org.opensearch.index.engine.VersionConflictEngineException) IOException(java.io.IOException)

Example 3 with ActionRunnable

use of org.opensearch.action.ActionRunnable in project OpenSearch by opensearch-project.

the class IndexShardOperationPermits method acquire.

private void acquire(final ActionListener<Releasable> onAcquired, final String executorOnDelay, final boolean forceExecution, final Object debugInfo, final StackTraceElement[] stackTrace) {
    if (closed) {
        onAcquired.onFailure(new IndexShardClosedException(shardId));
        return;
    }
    final Releasable releasable;
    try {
        synchronized (this) {
            if (queuedBlockOperations > 0) {
                final Supplier<StoredContext> contextSupplier = threadPool.getThreadContext().newRestorableContext(false);
                final ActionListener<Releasable> wrappedListener;
                if (executorOnDelay != null) {
                    wrappedListener = ActionListener.delegateFailure(new ContextPreservingActionListener<>(contextSupplier, onAcquired), (l, r) -> threadPool.executor(executorOnDelay).execute(new ActionRunnable<Releasable>(l) {

                        @Override
                        public boolean isForceExecution() {
                            return forceExecution;
                        }

                        @Override
                        protected void doRun() {
                            listener.onResponse(r);
                        }

                        @Override
                        public void onRejection(Exception e) {
                            IOUtils.closeWhileHandlingException(r);
                            super.onRejection(e);
                        }
                    }));
                } else {
                    wrappedListener = new ContextPreservingActionListener<>(contextSupplier, onAcquired);
                }
                delayedOperations.add(new DelayedOperation(wrappedListener, debugInfo, stackTrace));
                return;
            } else {
                releasable = acquire(debugInfo, stackTrace);
            }
        }
    } catch (final InterruptedException e) {
        onAcquired.onFailure(e);
        return;
    }
    // execute this outside the synchronized block!
    onAcquired.onResponse(releasable);
}
Also used : StoredContext(org.opensearch.common.util.concurrent.ThreadContext.StoredContext) ContextPreservingActionListener(org.opensearch.action.support.ContextPreservingActionListener) ActionRunnable(org.opensearch.action.ActionRunnable) AbstractRunnable(org.opensearch.common.util.concurrent.AbstractRunnable) ThreadPool(org.opensearch.threadpool.ThreadPool) TimeoutException(java.util.concurrent.TimeoutException) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) Releasable(org.opensearch.common.lease.Releasable) Supplier(java.util.function.Supplier) RunOnce(org.opensearch.common.util.concurrent.RunOnce) ArrayList(java.util.ArrayList) Assertions(org.opensearch.Assertions) Map(java.util.Map) ActionListener(org.opensearch.action.ActionListener) Semaphore(java.util.concurrent.Semaphore) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) ExceptionsHelper(org.opensearch.ExceptionsHelper) Collectors(java.util.stream.Collectors) Tuple(org.opensearch.common.collect.Tuple) IOUtils(org.opensearch.core.internal.io.IOUtils) TimeUnit(java.util.concurrent.TimeUnit) List(java.util.List) Closeable(java.io.Closeable) CheckedRunnable(org.opensearch.common.CheckedRunnable) Collections(java.util.Collections) TimeoutException(java.util.concurrent.TimeoutException) StoredContext(org.opensearch.common.util.concurrent.ThreadContext.StoredContext) ContextPreservingActionListener(org.opensearch.action.support.ContextPreservingActionListener) Releasable(org.opensearch.common.lease.Releasable)

Aggregations

Map (java.util.Map)3 ExceptionsHelper (org.opensearch.ExceptionsHelper)3 ActionListener (org.opensearch.action.ActionListener)3 ActionRunnable (org.opensearch.action.ActionRunnable)3 ThreadPool (org.opensearch.threadpool.ThreadPool)3 ArrayList (java.util.ArrayList)2 List (java.util.List)2 TimeUnit (java.util.concurrent.TimeUnit)2 LongSupplier (java.util.function.LongSupplier)2 Collectors (java.util.stream.Collectors)2 LogManager (org.apache.logging.log4j.LogManager)2 Logger (org.apache.logging.log4j.Logger)2 Assertions (org.opensearch.Assertions)2 DocWriteRequest (org.opensearch.action.DocWriteRequest)2 DocWriteResponse (org.opensearch.action.DocWriteResponse)2 IndexRequest (org.opensearch.action.index.IndexRequest)2 ActionFilters (org.opensearch.action.support.ActionFilters)2 UpdateRequest (org.opensearch.action.update.UpdateRequest)2 UpdateResponse (org.opensearch.action.update.UpdateResponse)2 ClusterState (org.opensearch.cluster.ClusterState)2