Search in sources :

Example 1 with DocWriteRequest

use of org.opensearch.action.DocWriteRequest in project OpenSearch by opensearch-project.

the class RequestConverters method bulk.

static Request bulk(BulkRequest bulkRequest) throws IOException {
    Request request = new Request(HttpPost.METHOD_NAME, "/_bulk");
    Params parameters = new Params();
    parameters.withTimeout(bulkRequest.timeout());
    parameters.withRefreshPolicy(bulkRequest.getRefreshPolicy());
    parameters.withPipeline(bulkRequest.pipeline());
    parameters.withRouting(bulkRequest.routing());
    // Bulk API only supports newline delimited JSON or Smile. Before executing
    // the bulk, we need to check that all requests have the same content-type
    // and this content-type is supported by the Bulk API.
    XContentType bulkContentType = null;
    for (int i = 0; i < bulkRequest.numberOfActions(); i++) {
        DocWriteRequest<?> action = bulkRequest.requests().get(i);
        DocWriteRequest.OpType opType = action.opType();
        if (opType == DocWriteRequest.OpType.INDEX || opType == DocWriteRequest.OpType.CREATE) {
            bulkContentType = enforceSameContentType((IndexRequest) action, bulkContentType);
        } else if (opType == DocWriteRequest.OpType.UPDATE) {
            UpdateRequest updateRequest = (UpdateRequest) action;
            if (updateRequest.doc() != null) {
                bulkContentType = enforceSameContentType(updateRequest.doc(), bulkContentType);
            }
            if (updateRequest.upsertRequest() != null) {
                bulkContentType = enforceSameContentType(updateRequest.upsertRequest(), bulkContentType);
            }
        }
    }
    if (bulkContentType == null) {
        bulkContentType = XContentType.JSON;
    }
    final byte separator = bulkContentType.xContent().streamSeparator();
    final ContentType requestContentType = createContentType(bulkContentType);
    ByteArrayOutputStream content = new ByteArrayOutputStream();
    for (DocWriteRequest<?> action : bulkRequest.requests()) {
        DocWriteRequest.OpType opType = action.opType();
        try (XContentBuilder metadata = XContentBuilder.builder(bulkContentType.xContent())) {
            metadata.startObject();
            {
                metadata.startObject(opType.getLowercase());
                if (Strings.hasLength(action.index())) {
                    metadata.field("_index", action.index());
                }
                if (Strings.hasLength(action.id())) {
                    metadata.field("_id", action.id());
                }
                if (Strings.hasLength(action.routing())) {
                    metadata.field("routing", action.routing());
                }
                if (action.version() != Versions.MATCH_ANY) {
                    metadata.field("version", action.version());
                }
                VersionType versionType = action.versionType();
                if (versionType != VersionType.INTERNAL) {
                    if (versionType == VersionType.EXTERNAL) {
                        metadata.field("version_type", "external");
                    } else if (versionType == VersionType.EXTERNAL_GTE) {
                        metadata.field("version_type", "external_gte");
                    }
                }
                if (action.ifSeqNo() != SequenceNumbers.UNASSIGNED_SEQ_NO) {
                    metadata.field("if_seq_no", action.ifSeqNo());
                    metadata.field("if_primary_term", action.ifPrimaryTerm());
                }
                if (opType == DocWriteRequest.OpType.INDEX || opType == DocWriteRequest.OpType.CREATE) {
                    IndexRequest indexRequest = (IndexRequest) action;
                    if (Strings.hasLength(indexRequest.getPipeline())) {
                        metadata.field("pipeline", indexRequest.getPipeline());
                    }
                } else if (opType == DocWriteRequest.OpType.UPDATE) {
                    UpdateRequest updateRequest = (UpdateRequest) action;
                    if (updateRequest.retryOnConflict() > 0) {
                        metadata.field("retry_on_conflict", updateRequest.retryOnConflict());
                    }
                    if (updateRequest.fetchSource() != null) {
                        metadata.field("_source", updateRequest.fetchSource());
                    }
                }
                metadata.endObject();
            }
            metadata.endObject();
            BytesRef metadataSource = BytesReference.bytes(metadata).toBytesRef();
            content.write(metadataSource.bytes, metadataSource.offset, metadataSource.length);
            content.write(separator);
        }
        BytesRef source = null;
        if (opType == DocWriteRequest.OpType.INDEX || opType == DocWriteRequest.OpType.CREATE) {
            IndexRequest indexRequest = (IndexRequest) action;
            BytesReference indexSource = indexRequest.source();
            XContentType indexXContentType = indexRequest.getContentType();
            try (XContentParser parser = XContentHelper.createParser(/*
                         * EMPTY and THROW are fine here because we just call
                         * copyCurrentStructure which doesn't touch the
                         * registry or deprecation.
                         */
            NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, indexSource, indexXContentType)) {
                try (XContentBuilder builder = XContentBuilder.builder(bulkContentType.xContent())) {
                    builder.copyCurrentStructure(parser);
                    source = BytesReference.bytes(builder).toBytesRef();
                }
            }
        } else if (opType == DocWriteRequest.OpType.UPDATE) {
            source = XContentHelper.toXContent((UpdateRequest) action, bulkContentType, false).toBytesRef();
        }
        if (source != null) {
            content.write(source.bytes, source.offset, source.length);
            content.write(separator);
        }
    }
    request.addParameters(parameters.asMap());
    request.setEntity(new NByteArrayEntity(content.toByteArray(), 0, content.size(), requestContentType));
    return request;
}
Also used : BytesReference(org.opensearch.common.bytes.BytesReference) ContentType(org.apache.http.entity.ContentType) XContentType(org.opensearch.common.xcontent.XContentType) UpdateRequest(org.opensearch.action.update.UpdateRequest) BulkRequest(org.opensearch.action.bulk.BulkRequest) WriteRequest(org.opensearch.action.support.WriteRequest) AbstractBulkByScrollRequest(org.opensearch.index.reindex.AbstractBulkByScrollRequest) GetStoredScriptRequest(org.opensearch.action.admin.cluster.storedscripts.GetStoredScriptRequest) AnalyzeRequest(org.opensearch.client.indices.AnalyzeRequest) DeleteRequest(org.opensearch.action.delete.DeleteRequest) CountRequest(org.opensearch.client.core.CountRequest) TermVectorsRequest(org.opensearch.client.core.TermVectorsRequest) FieldCapabilitiesRequest(org.opensearch.action.fieldcaps.FieldCapabilitiesRequest) UpdateRequest(org.opensearch.action.update.UpdateRequest) SearchTemplateRequest(org.opensearch.script.mustache.SearchTemplateRequest) RankEvalRequest(org.opensearch.index.rankeval.RankEvalRequest) GetSourceRequest(org.opensearch.client.core.GetSourceRequest) MultiSearchRequest(org.opensearch.action.search.MultiSearchRequest) DocWriteRequest(org.opensearch.action.DocWriteRequest) SearchScrollRequest(org.opensearch.action.search.SearchScrollRequest) ExplainRequest(org.opensearch.action.explain.ExplainRequest) SearchRequest(org.opensearch.action.search.SearchRequest) DeleteStoredScriptRequest(org.opensearch.action.admin.cluster.storedscripts.DeleteStoredScriptRequest) GetRequest(org.opensearch.action.get.GetRequest) MultiSearchTemplateRequest(org.opensearch.script.mustache.MultiSearchTemplateRequest) PutStoredScriptRequest(org.opensearch.action.admin.cluster.storedscripts.PutStoredScriptRequest) UpdateByQueryRequest(org.opensearch.index.reindex.UpdateByQueryRequest) MultiTermVectorsRequest(org.opensearch.client.core.MultiTermVectorsRequest) ClearScrollRequest(org.opensearch.action.search.ClearScrollRequest) DeleteByQueryRequest(org.opensearch.index.reindex.DeleteByQueryRequest) IndexRequest(org.opensearch.action.index.IndexRequest) ClusterHealthRequest(org.opensearch.action.admin.cluster.health.ClusterHealthRequest) ReindexRequest(org.opensearch.index.reindex.ReindexRequest) MultiGetRequest(org.opensearch.action.get.MultiGetRequest) ByteArrayOutputStream(java.io.ByteArrayOutputStream) IndexRequest(org.opensearch.action.index.IndexRequest) VersionType(org.opensearch.index.VersionType) XContentType(org.opensearch.common.xcontent.XContentType) NByteArrayEntity(org.apache.http.nio.entity.NByteArrayEntity) DocWriteRequest(org.opensearch.action.DocWriteRequest) XContentBuilder(org.opensearch.common.xcontent.XContentBuilder) BytesRef(org.apache.lucene.util.BytesRef) XContentParser(org.opensearch.common.xcontent.XContentParser)

Example 2 with DocWriteRequest

use of org.opensearch.action.DocWriteRequest in project OpenSearch by opensearch-project.

the class RequestConvertersTests method testBulk.

public void testBulk() throws IOException {
    Map<String, String> expectedParams = new HashMap<>();
    BulkRequest bulkRequest = new BulkRequest();
    if (randomBoolean()) {
        String timeout = randomTimeValue();
        bulkRequest.timeout(timeout);
        expectedParams.put("timeout", timeout);
    } else {
        expectedParams.put("timeout", BulkShardRequest.DEFAULT_TIMEOUT.getStringRep());
    }
    setRandomRefreshPolicy(bulkRequest::setRefreshPolicy, expectedParams);
    XContentType xContentType = randomFrom(XContentType.JSON, XContentType.SMILE);
    int nbItems = randomIntBetween(10, 100);
    DocWriteRequest<?>[] requests = new DocWriteRequest<?>[nbItems];
    for (int i = 0; i < nbItems; i++) {
        String index = randomAlphaOfLength(5);
        String id = randomAlphaOfLength(5);
        BytesReference source = RandomObjects.randomSource(random(), xContentType);
        DocWriteRequest.OpType opType = randomFrom(DocWriteRequest.OpType.values());
        DocWriteRequest<?> docWriteRequest;
        if (opType == DocWriteRequest.OpType.INDEX) {
            IndexRequest indexRequest = new IndexRequest(index).id(id).source(source, xContentType);
            docWriteRequest = indexRequest;
            if (randomBoolean()) {
                indexRequest.setPipeline(randomAlphaOfLength(5));
            }
        } else if (opType == DocWriteRequest.OpType.CREATE) {
            IndexRequest createRequest = new IndexRequest(index).id(id).source(source, xContentType).create(true);
            docWriteRequest = createRequest;
        } else if (opType == DocWriteRequest.OpType.UPDATE) {
            final UpdateRequest updateRequest = new UpdateRequest(index, id).doc(new IndexRequest().source(source, xContentType));
            docWriteRequest = updateRequest;
            if (randomBoolean()) {
                updateRequest.retryOnConflict(randomIntBetween(1, 5));
            }
            if (randomBoolean()) {
                randomizeFetchSourceContextParams(updateRequest::fetchSource, new HashMap<>());
            }
        } else if (opType == DocWriteRequest.OpType.DELETE) {
            docWriteRequest = new DeleteRequest(index, id);
        } else {
            throw new UnsupportedOperationException("optype [" + opType + "] not supported");
        }
        if (randomBoolean()) {
            docWriteRequest.routing(randomAlphaOfLength(10));
        }
        if (opType != DocWriteRequest.OpType.UPDATE && randomBoolean()) {
            docWriteRequest.setIfSeqNo(randomNonNegativeLong());
            docWriteRequest.setIfPrimaryTerm(randomLongBetween(1, 200));
        }
        requests[i] = docWriteRequest;
    }
    bulkRequest.add(requests);
    Request request = RequestConverters.bulk(bulkRequest);
    assertEquals("/_bulk", request.getEndpoint());
    assertEquals(expectedParams, request.getParameters());
    assertEquals(HttpPost.METHOD_NAME, request.getMethod());
    assertEquals(xContentType.mediaTypeWithoutParameters(), request.getEntity().getContentType().getValue());
    byte[] content = new byte[(int) request.getEntity().getContentLength()];
    try (InputStream inputStream = request.getEntity().getContent()) {
        Streams.readFully(inputStream, content);
    }
    BulkRequest parsedBulkRequest = new BulkRequest();
    parsedBulkRequest.add(content, 0, content.length, xContentType);
    assertEquals(bulkRequest.numberOfActions(), parsedBulkRequest.numberOfActions());
    for (int i = 0; i < bulkRequest.numberOfActions(); i++) {
        DocWriteRequest<?> originalRequest = bulkRequest.requests().get(i);
        DocWriteRequest<?> parsedRequest = parsedBulkRequest.requests().get(i);
        assertEquals(originalRequest.opType(), parsedRequest.opType());
        assertEquals(originalRequest.index(), parsedRequest.index());
        assertEquals(originalRequest.id(), parsedRequest.id());
        assertEquals(originalRequest.routing(), parsedRequest.routing());
        assertEquals(originalRequest.version(), parsedRequest.version());
        assertEquals(originalRequest.versionType(), parsedRequest.versionType());
        assertEquals(originalRequest.ifSeqNo(), parsedRequest.ifSeqNo());
        assertEquals(originalRequest.ifPrimaryTerm(), parsedRequest.ifPrimaryTerm());
        DocWriteRequest.OpType opType = originalRequest.opType();
        if (opType == DocWriteRequest.OpType.INDEX) {
            IndexRequest indexRequest = (IndexRequest) originalRequest;
            IndexRequest parsedIndexRequest = (IndexRequest) parsedRequest;
            assertEquals(indexRequest.getPipeline(), parsedIndexRequest.getPipeline());
            assertToXContentEquivalent(indexRequest.source(), parsedIndexRequest.source(), xContentType);
        } else if (opType == DocWriteRequest.OpType.UPDATE) {
            UpdateRequest updateRequest = (UpdateRequest) originalRequest;
            UpdateRequest parsedUpdateRequest = (UpdateRequest) parsedRequest;
            assertEquals(updateRequest.retryOnConflict(), parsedUpdateRequest.retryOnConflict());
            assertEquals(updateRequest.fetchSource(), parsedUpdateRequest.fetchSource());
            if (updateRequest.doc() != null) {
                assertToXContentEquivalent(updateRequest.doc().source(), parsedUpdateRequest.doc().source(), xContentType);
            } else {
                assertNull(parsedUpdateRequest.doc());
            }
        }
    }
}
Also used : BytesReference(org.opensearch.common.bytes.BytesReference) HashMap(java.util.HashMap) UpdateRequest(org.opensearch.action.update.UpdateRequest) InputStream(java.io.InputStream) RandomSearchRequestGenerator.randomSearchRequest(org.opensearch.search.RandomSearchRequestGenerator.randomSearchRequest) MasterNodeRequest(org.opensearch.action.support.master.MasterNodeRequest) WriteRequest(org.opensearch.action.support.WriteRequest) AbstractBulkByScrollRequest(org.opensearch.index.reindex.AbstractBulkByScrollRequest) RatedRequest(org.opensearch.index.rankeval.RatedRequest) DeleteRequest(org.opensearch.action.delete.DeleteRequest) TermVectorsRequest(org.opensearch.client.core.TermVectorsRequest) AcknowledgedRequest(org.opensearch.action.support.master.AcknowledgedRequest) FieldCapabilitiesRequest(org.opensearch.action.fieldcaps.FieldCapabilitiesRequest) UpdateRequest(org.opensearch.action.update.UpdateRequest) GetSourceRequest(org.opensearch.client.core.GetSourceRequest) MultiSearchRequest(org.opensearch.action.search.MultiSearchRequest) DocWriteRequest(org.opensearch.action.DocWriteRequest) SearchScrollRequest(org.opensearch.action.search.SearchScrollRequest) ExplainRequest(org.opensearch.action.explain.ExplainRequest) SearchRequest(org.opensearch.action.search.SearchRequest) PutStoredScriptRequest(org.opensearch.action.admin.cluster.storedscripts.PutStoredScriptRequest) DeleteByQueryRequest(org.opensearch.index.reindex.DeleteByQueryRequest) MultiGetRequest(org.opensearch.action.get.MultiGetRequest) BulkRequest(org.opensearch.action.bulk.BulkRequest) GetStoredScriptRequest(org.opensearch.action.admin.cluster.storedscripts.GetStoredScriptRequest) ReplicationRequest(org.opensearch.action.support.replication.ReplicationRequest) AnalyzeRequest(org.opensearch.client.indices.AnalyzeRequest) CountRequest(org.opensearch.client.core.CountRequest) BulkShardRequest(org.opensearch.action.bulk.BulkShardRequest) SearchTemplateRequest(org.opensearch.script.mustache.SearchTemplateRequest) RankEvalRequest(org.opensearch.index.rankeval.RankEvalRequest) DeleteStoredScriptRequest(org.opensearch.action.admin.cluster.storedscripts.DeleteStoredScriptRequest) GetRequest(org.opensearch.action.get.GetRequest) MultiSearchTemplateRequest(org.opensearch.script.mustache.MultiSearchTemplateRequest) UpdateByQueryRequest(org.opensearch.index.reindex.UpdateByQueryRequest) MultiTermVectorsRequest(org.opensearch.client.core.MultiTermVectorsRequest) ClearScrollRequest(org.opensearch.action.search.ClearScrollRequest) IndexRequest(org.opensearch.action.index.IndexRequest) ReindexRequest(org.opensearch.index.reindex.ReindexRequest) IndexRequest(org.opensearch.action.index.IndexRequest) XContentType(org.opensearch.common.xcontent.XContentType) BulkRequest(org.opensearch.action.bulk.BulkRequest) DocWriteRequest(org.opensearch.action.DocWriteRequest) DeleteRequest(org.opensearch.action.delete.DeleteRequest)

Example 3 with DocWriteRequest

use of org.opensearch.action.DocWriteRequest in project OpenSearch by opensearch-project.

the class TransportBulkAction method prohibitAppendWritesInBackingIndices.

static void prohibitAppendWritesInBackingIndices(DocWriteRequest<?> writeRequest, Metadata metadata) {
    IndexAbstraction indexAbstraction = metadata.getIndicesLookup().get(writeRequest.index());
    if (indexAbstraction == null) {
        return;
    }
    if (indexAbstraction.getType() != IndexAbstraction.Type.CONCRETE_INDEX) {
        return;
    }
    if (indexAbstraction.getParentDataStream() == null) {
        return;
    }
    DataStream dataStream = indexAbstraction.getParentDataStream().getDataStream();
    // At this point with write op is targeting a backing index of a data stream directly,
    // so checking if write op is append-only and if so fail.
    // (Updates and deletes are allowed to target a backing index)
    DocWriteRequest.OpType opType = writeRequest.opType();
    // that an engine level change is needed and for now this check is sufficient.)
    if (opType == DocWriteRequest.OpType.CREATE) {
        throw new IllegalArgumentException("index request with op_type=create targeting backing indices is disallowed, " + "target corresponding data stream [" + dataStream.getName() + "] instead");
    }
    if (opType == DocWriteRequest.OpType.INDEX && writeRequest.ifPrimaryTerm() == UNASSIGNED_PRIMARY_TERM && writeRequest.ifSeqNo() == UNASSIGNED_SEQ_NO) {
        throw new IllegalArgumentException("index request with op_type=index and no if_primary_term and if_seq_no set " + "targeting backing indices is disallowed, target corresponding data stream [" + dataStream.getName() + "] instead");
    }
}
Also used : IndexAbstraction(org.opensearch.cluster.metadata.IndexAbstraction) DataStream(org.opensearch.cluster.metadata.DataStream) DocWriteRequest(org.opensearch.action.DocWriteRequest)

Example 4 with DocWriteRequest

use of org.opensearch.action.DocWriteRequest in project OpenSearch by opensearch-project.

the class TransportBulkAction method doInternalExecute.

protected void doInternalExecute(Task task, BulkRequest bulkRequest, String executorName, ActionListener<BulkResponse> listener) {
    final long startTime = relativeTime();
    final AtomicArray<BulkItemResponse> responses = new AtomicArray<>(bulkRequest.requests.size());
    boolean hasIndexRequestsWithPipelines = false;
    final Metadata metadata = clusterService.state().getMetadata();
    final Version minNodeVersion = clusterService.state().getNodes().getMinNodeVersion();
    for (DocWriteRequest<?> actionRequest : bulkRequest.requests) {
        IndexRequest indexRequest = getIndexWriteRequest(actionRequest);
        if (indexRequest != null) {
            // Each index request needs to be evaluated, because this method also modifies the IndexRequest
            boolean indexRequestHasPipeline = IngestService.resolvePipelines(actionRequest, indexRequest, metadata);
            hasIndexRequestsWithPipelines |= indexRequestHasPipeline;
        }
        if (actionRequest instanceof IndexRequest) {
            IndexRequest ir = (IndexRequest) actionRequest;
            ir.checkAutoIdWithOpTypeCreateSupportedByVersion(minNodeVersion);
            if (ir.getAutoGeneratedTimestamp() != IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP) {
                throw new IllegalArgumentException("autoGeneratedTimestamp should not be set externally");
            }
        }
    }
    if (hasIndexRequestsWithPipelines) {
        // this path is never taken.
        try {
            if (Assertions.ENABLED) {
                final boolean arePipelinesResolved = bulkRequest.requests().stream().map(TransportBulkAction::getIndexWriteRequest).filter(Objects::nonNull).allMatch(IndexRequest::isPipelineResolved);
                assert arePipelinesResolved : bulkRequest;
            }
            if (clusterService.localNode().isIngestNode()) {
                processBulkIndexIngestRequest(task, bulkRequest, executorName, listener);
            } else {
                ingestForwarder.forwardIngestRequest(BulkAction.INSTANCE, bulkRequest, listener);
            }
        } catch (Exception e) {
            listener.onFailure(e);
        }
        return;
    }
    final boolean includesSystem = includesSystem(bulkRequest, clusterService.state().metadata().getIndicesLookup(), systemIndices);
    if (includesSystem || needToCheck()) {
        // Attempt to create all the indices that we're going to need during the bulk before we start.
        // Step 1: collect all the indices in the request
        final Map<String, Boolean> indices = bulkRequest.requests.stream().filter(request -> request.opType() != DocWriteRequest.OpType.DELETE || request.versionType() == VersionType.EXTERNAL || request.versionType() == VersionType.EXTERNAL_GTE).collect(Collectors.toMap(DocWriteRequest::index, DocWriteRequest::isRequireAlias, (v1, v2) -> v1 || v2));
        /* Step 2: filter that to indices that don't exist and we can create. At the same time build a map of indices we can't create
             * that we'll use when we try to run the requests. */
        final Map<String, IndexNotFoundException> indicesThatCannotBeCreated = new HashMap<>();
        Set<String> autoCreateIndices = new HashSet<>();
        ClusterState state = clusterService.state();
        for (Map.Entry<String, Boolean> indexAndFlag : indices.entrySet()) {
            boolean shouldAutoCreate;
            final String index = indexAndFlag.getKey();
            try {
                shouldAutoCreate = shouldAutoCreate(index, state);
            } catch (IndexNotFoundException e) {
                shouldAutoCreate = false;
                indicesThatCannotBeCreated.put(index, e);
            }
            // We should only auto create if we are not requiring it to be an alias
            if (shouldAutoCreate && (indexAndFlag.getValue() == false)) {
                autoCreateIndices.add(index);
            }
        }
        // Step 3: create all the indices that are missing, if there are any missing. start the bulk after all the creates come back.
        if (autoCreateIndices.isEmpty()) {
            executeBulk(task, bulkRequest, startTime, listener, responses, indicesThatCannotBeCreated);
        } else {
            final AtomicInteger counter = new AtomicInteger(autoCreateIndices.size());
            for (String index : autoCreateIndices) {
                createIndex(index, bulkRequest.timeout(), minNodeVersion, new ActionListener<CreateIndexResponse>() {

                    @Override
                    public void onResponse(CreateIndexResponse result) {
                        if (counter.decrementAndGet() == 0) {
                            threadPool.executor(executorName).execute(new ActionRunnable<BulkResponse>(listener) {

                                @Override
                                protected void doRun() {
                                    executeBulk(task, bulkRequest, startTime, listener, responses, indicesThatCannotBeCreated);
                                }
                            });
                        }
                    }

                    @Override
                    public void onFailure(Exception e) {
                        if (!(ExceptionsHelper.unwrapCause(e) instanceof ResourceAlreadyExistsException)) {
                            // fail all requests involving this index, if create didn't work
                            for (int i = 0; i < bulkRequest.requests.size(); i++) {
                                DocWriteRequest<?> request = bulkRequest.requests.get(i);
                                if (request != null && setResponseFailureIfIndexMatches(responses, i, request, index, e)) {
                                    bulkRequest.requests.set(i, null);
                                }
                            }
                        }
                        if (counter.decrementAndGet() == 0) {
                            final ActionListener<BulkResponse> wrappedListener = ActionListener.wrap(listener::onResponse, inner -> {
                                inner.addSuppressed(e);
                                listener.onFailure(inner);
                            });
                            threadPool.executor(executorName).execute(new ActionRunnable<BulkResponse>(wrappedListener) {

                                @Override
                                protected void doRun() {
                                    executeBulk(task, bulkRequest, startTime, wrappedListener, responses, indicesThatCannotBeCreated);
                                }

                                @Override
                                public void onRejection(Exception rejectedException) {
                                    rejectedException.addSuppressed(e);
                                    super.onRejection(rejectedException);
                                }
                            });
                        }
                    }
                });
            }
        }
    } else {
        executeBulk(task, bulkRequest, startTime, listener, responses, emptyMap());
    }
}
Also used : SequenceNumbers(org.opensearch.index.seqno.SequenceNumbers) IndexAbstraction(org.opensearch.cluster.metadata.IndexAbstraction) Metadata(org.opensearch.cluster.metadata.Metadata) LongSupplier(java.util.function.LongSupplier) DataStream(org.opensearch.cluster.metadata.DataStream) Version(org.opensearch.Version) TransportUpdateAction(org.opensearch.action.update.TransportUpdateAction) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Locale(java.util.Locale) Assertions(org.opensearch.Assertions) Map(java.util.Map) NodeClosedException(org.opensearch.node.NodeClosedException) AutoCreateAction(org.opensearch.action.admin.indices.create.AutoCreateAction) Inject(org.opensearch.common.inject.Inject) ActionListener(org.opensearch.action.ActionListener) AtomicIntegerArray(java.util.concurrent.atomic.AtomicIntegerArray) CreateIndexRequest(org.opensearch.action.admin.indices.create.CreateIndexRequest) TimeValue(org.opensearch.common.unit.TimeValue) NodeClient(org.opensearch.client.node.NodeClient) Index(org.opensearch.index.Index) IndexingPressureService(org.opensearch.index.IndexingPressureService) OpenSearchParseException(org.opensearch.OpenSearchParseException) ExceptionsHelper(org.opensearch.ExceptionsHelper) ClusterBlockException(org.opensearch.cluster.block.ClusterBlockException) Set(java.util.Set) Task(org.opensearch.tasks.Task) TransportService(org.opensearch.transport.TransportService) Collectors(java.util.stream.Collectors) Objects(java.util.Objects) ActionFilters(org.opensearch.action.support.ActionFilters) VersionType(org.opensearch.index.VersionType) List(java.util.List) Logger(org.apache.logging.log4j.Logger) SparseFixedBitSet(org.apache.lucene.util.SparseFixedBitSet) EXCLUDED_DATA_STREAMS_KEY(org.opensearch.cluster.metadata.IndexNameExpressionResolver.EXCLUDED_DATA_STREAMS_KEY) ResourceAlreadyExistsException(org.opensearch.ResourceAlreadyExistsException) DocWriteResponse(org.opensearch.action.DocWriteResponse) UpdateRequest(org.opensearch.action.update.UpdateRequest) SortedMap(java.util.SortedMap) IndexNameExpressionResolver(org.opensearch.cluster.metadata.IndexNameExpressionResolver) Names(org.opensearch.threadpool.ThreadPool.Names) MappingMetadata(org.opensearch.cluster.metadata.MappingMetadata) HandledTransportAction(org.opensearch.action.support.HandledTransportAction) IndexMetadata(org.opensearch.cluster.metadata.IndexMetadata) ActionRunnable(org.opensearch.action.ActionRunnable) UpdateResponse(org.opensearch.action.update.UpdateResponse) ThreadPool(org.opensearch.threadpool.ThreadPool) RoutingMissingException(org.opensearch.action.RoutingMissingException) DocWriteRequest(org.opensearch.action.DocWriteRequest) HashMap(java.util.HashMap) Releasable(org.opensearch.common.lease.Releasable) ArrayList(java.util.ArrayList) AutoCreateIndex(org.opensearch.action.support.AutoCreateIndex) HashSet(java.util.HashSet) ClusterState(org.opensearch.cluster.ClusterState) UNASSIGNED_SEQ_NO(org.opensearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO) LegacyESVersion(org.opensearch.LegacyESVersion) IndexClosedException(org.opensearch.indices.IndexClosedException) ClusterStateObserver(org.opensearch.cluster.ClusterStateObserver) Collections.emptyMap(java.util.Collections.emptyMap) IngestService(org.opensearch.ingest.IngestService) Iterator(java.util.Iterator) IngestActionForwarder(org.opensearch.action.ingest.IngestActionForwarder) ClusterBlockLevel(org.opensearch.cluster.block.ClusterBlockLevel) IndexNotFoundException(org.opensearch.index.IndexNotFoundException) CreateIndexResponse(org.opensearch.action.admin.indices.create.CreateIndexResponse) UNASSIGNED_PRIMARY_TERM(org.opensearch.index.seqno.SequenceNumbers.UNASSIGNED_PRIMARY_TERM) ShardId(org.opensearch.index.shard.ShardId) TimeUnit(java.util.concurrent.TimeUnit) SystemIndices(org.opensearch.indices.SystemIndices) AtomicArray(org.opensearch.common.util.concurrent.AtomicArray) ClusterService(org.opensearch.cluster.service.ClusterService) IndexRequest(org.opensearch.action.index.IndexRequest) LogManager(org.apache.logging.log4j.LogManager) AtomicArray(org.opensearch.common.util.concurrent.AtomicArray) ActionRunnable(org.opensearch.action.ActionRunnable) HashMap(java.util.HashMap) Metadata(org.opensearch.cluster.metadata.Metadata) MappingMetadata(org.opensearch.cluster.metadata.MappingMetadata) IndexMetadata(org.opensearch.cluster.metadata.IndexMetadata) CreateIndexRequest(org.opensearch.action.admin.indices.create.CreateIndexRequest) IndexRequest(org.opensearch.action.index.IndexRequest) Version(org.opensearch.Version) LegacyESVersion(org.opensearch.LegacyESVersion) CreateIndexResponse(org.opensearch.action.admin.indices.create.CreateIndexResponse) HashSet(java.util.HashSet) ClusterState(org.opensearch.cluster.ClusterState) ResourceAlreadyExistsException(org.opensearch.ResourceAlreadyExistsException) NodeClosedException(org.opensearch.node.NodeClosedException) OpenSearchParseException(org.opensearch.OpenSearchParseException) ClusterBlockException(org.opensearch.cluster.block.ClusterBlockException) ResourceAlreadyExistsException(org.opensearch.ResourceAlreadyExistsException) RoutingMissingException(org.opensearch.action.RoutingMissingException) IndexClosedException(org.opensearch.indices.IndexClosedException) IndexNotFoundException(org.opensearch.index.IndexNotFoundException) ActionListener(org.opensearch.action.ActionListener) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) IndexNotFoundException(org.opensearch.index.IndexNotFoundException) DocWriteRequest(org.opensearch.action.DocWriteRequest) Map(java.util.Map) SortedMap(java.util.SortedMap) HashMap(java.util.HashMap) Collections.emptyMap(java.util.Collections.emptyMap)

Example 5 with DocWriteRequest

use of org.opensearch.action.DocWriteRequest in project OpenSearch by opensearch-project.

the class TransportShardBulkAction method performOnPrimary.

public static void performOnPrimary(BulkShardRequest request, IndexShard primary, UpdateHelper updateHelper, LongSupplier nowInMillisSupplier, MappingUpdatePerformer mappingUpdater, Consumer<ActionListener<Void>> waitForMappingUpdate, ActionListener<PrimaryResult<BulkShardRequest, BulkShardResponse>> listener, ThreadPool threadPool, String executorName) {
    new ActionRunnable<PrimaryResult<BulkShardRequest, BulkShardResponse>>(listener) {

        private final Executor executor = threadPool.executor(executorName);

        private final BulkPrimaryExecutionContext context = new BulkPrimaryExecutionContext(request, primary);

        @Override
        protected void doRun() throws Exception {
            while (context.hasMoreOperationsToExecute()) {
                if (executeBulkItemRequest(context, updateHelper, nowInMillisSupplier, mappingUpdater, waitForMappingUpdate, ActionListener.wrap(v -> executor.execute(this), this::onRejection)) == false) {
                    // so we just break out here.
                    return;
                }
                // either completed and moved to next or reset
                assert context.isInitial();
            }
            // We're done, there's no more operations to execute so we resolve the wrapped listener
            finishRequest();
        }

        @Override
        public void onRejection(Exception e) {
            // We must finish the outstanding request. Finishing the outstanding request can include
            // refreshing and fsyncing. Therefore, we must force execution on the WRITE thread.
            executor.execute(new ActionRunnable<PrimaryResult<BulkShardRequest, BulkShardResponse>>(listener) {

                @Override
                protected void doRun() {
                    // Fail all operations after a bulk rejection hit an action that waited for a mapping update and finish the request
                    while (context.hasMoreOperationsToExecute()) {
                        context.setRequestToExecute(context.getCurrent());
                        final DocWriteRequest<?> docWriteRequest = context.getRequestToExecute();
                        onComplete(exceptionToResult(e, primary, docWriteRequest.opType() == DocWriteRequest.OpType.DELETE, docWriteRequest.version()), context, null);
                    }
                    finishRequest();
                }

                @Override
                public boolean isForceExecution() {
                    return true;
                }
            });
        }

        private void finishRequest() {
            ActionListener.completeWith(listener, () -> new WritePrimaryResult<>(context.getBulkShardRequest(), context.buildShardResponse(), context.getLocationToSync(), null, context.getPrimary(), logger));
        }
    }.run();
}
Also used : SequenceNumbers(org.opensearch.index.seqno.SequenceNumbers) LongSupplier(java.util.function.LongSupplier) IndexResponse(org.opensearch.action.index.IndexResponse) ToXContent(org.opensearch.common.xcontent.ToXContent) MapperService(org.opensearch.index.mapper.MapperService) Map(java.util.Map) NodeClosedException(org.opensearch.node.NodeClosedException) Inject(org.opensearch.common.inject.Inject) ActionListener(org.opensearch.action.ActionListener) ShardStateAction(org.opensearch.cluster.action.shard.ShardStateAction) DeleteRequest(org.opensearch.action.delete.DeleteRequest) MapperException(org.opensearch.index.mapper.MapperException) TimeValue(org.opensearch.common.unit.TimeValue) IndexingPressureService(org.opensearch.index.IndexingPressureService) GetResult(org.opensearch.index.get.GetResult) IndicesService(org.opensearch.indices.IndicesService) TransportRequestOptions(org.opensearch.transport.TransportRequestOptions) ExceptionsHelper(org.opensearch.ExceptionsHelper) Settings(org.opensearch.common.settings.Settings) TransportService(org.opensearch.transport.TransportService) Tuple(org.opensearch.common.collect.Tuple) Engine(org.opensearch.index.engine.Engine) ActionFilters(org.opensearch.action.support.ActionFilters) Logger(org.apache.logging.log4j.Logger) DocWriteResponse(org.opensearch.action.DocWriteResponse) UpdateRequest(org.opensearch.action.update.UpdateRequest) XContentType(org.opensearch.common.xcontent.XContentType) Names(org.opensearch.threadpool.ThreadPool.Names) MappingMetadata(org.opensearch.cluster.metadata.MappingMetadata) MessageSupplier(org.apache.logging.log4j.util.MessageSupplier) BytesReference(org.opensearch.common.bytes.BytesReference) IndexMetadata(org.opensearch.cluster.metadata.IndexMetadata) ActionRunnable(org.opensearch.action.ActionRunnable) UpdateResponse(org.opensearch.action.update.UpdateResponse) ThreadPool(org.opensearch.threadpool.ThreadPool) CompressedXContent(org.opensearch.common.compress.CompressedXContent) DocWriteRequest(org.opensearch.action.DocWriteRequest) ParameterizedMessage(org.apache.logging.log4j.message.ParameterizedMessage) Function(java.util.function.Function) MappingUpdatedAction(org.opensearch.cluster.action.index.MappingUpdatedAction) SourceToParse(org.opensearch.index.mapper.SourceToParse) ClusterState(org.opensearch.cluster.ClusterState) IndexShard(org.opensearch.index.shard.IndexShard) DeleteResponse(org.opensearch.action.delete.DeleteResponse) Translog(org.opensearch.index.translog.Translog) TransportWriteAction(org.opensearch.action.support.replication.TransportWriteAction) ClusterStateObserver(org.opensearch.cluster.ClusterStateObserver) StreamInput(org.opensearch.common.io.stream.StreamInput) UpdateHelper(org.opensearch.action.update.UpdateHelper) Executor(java.util.concurrent.Executor) VersionConflictEngineException(org.opensearch.index.engine.VersionConflictEngineException) IOException(java.io.IOException) TransportReplicationAction(org.opensearch.action.support.replication.TransportReplicationAction) XContentHelper(org.opensearch.common.xcontent.XContentHelper) ShardId(org.opensearch.index.shard.ShardId) Consumer(java.util.function.Consumer) SystemIndices(org.opensearch.indices.SystemIndices) ClusterService(org.opensearch.cluster.service.ClusterService) IndexRequest(org.opensearch.action.index.IndexRequest) LogManager(org.apache.logging.log4j.LogManager) ActionRunnable(org.opensearch.action.ActionRunnable) Executor(java.util.concurrent.Executor) NodeClosedException(org.opensearch.node.NodeClosedException) MapperException(org.opensearch.index.mapper.MapperException) VersionConflictEngineException(org.opensearch.index.engine.VersionConflictEngineException) IOException(java.io.IOException)

Aggregations

DocWriteRequest (org.opensearch.action.DocWriteRequest)18 IndexRequest (org.opensearch.action.index.IndexRequest)13 UpdateRequest (org.opensearch.action.update.UpdateRequest)13 DeleteRequest (org.opensearch.action.delete.DeleteRequest)9 ClusterState (org.opensearch.cluster.ClusterState)7 Map (java.util.Map)5 Version (org.opensearch.Version)5 ActionListener (org.opensearch.action.ActionListener)5 IndexMetadata (org.opensearch.cluster.metadata.IndexMetadata)5 ClusterService (org.opensearch.cluster.service.ClusterService)5 ThreadPool (org.opensearch.threadpool.ThreadPool)5 Names (org.opensearch.threadpool.ThreadPool.Names)5 ArrayList (java.util.ArrayList)4 Collections (java.util.Collections)4 Collections.emptyMap (java.util.Collections.emptyMap)4 AtomicInteger (java.util.concurrent.atomic.AtomicInteger)4 Mockito.any (org.mockito.Mockito.any)4 Mockito.eq (org.mockito.Mockito.eq)4 IndexResponse (org.opensearch.action.index.IndexResponse)4 ActionFilters (org.opensearch.action.support.ActionFilters)4