Examples with DocWriteRequest org.opensearch.action.DocWriteRequest used on opensource projects

Example 11 with DocWriteRequest

use of org.opensearch.action.DocWriteRequest in project OpenSearch by opensearch-project.

the class BulkPrimaryExecutionContextTests method generateRandomRequest.

private BulkShardRequest generateRandomRequest() {
    BulkItemRequest[] items = new BulkItemRequest[randomInt(20)];
    for (int i = 0; i < items.length; i++) {
        final DocWriteRequest request;
        switch(randomFrom(DocWriteRequest.OpType.values())) {
            case INDEX:
                request = new IndexRequest("index").id("id_" + i);
                break;
            case CREATE:
                request = new IndexRequest("index").id("id_" + i).create(true);
                break;
            case UPDATE:
                request = new UpdateRequest("index", "id_" + i);
                break;
            case DELETE:
                request = new DeleteRequest("index", "id_" + i);
                break;
            default:
                throw new AssertionError("unknown type");
        }
        items[i] = new BulkItemRequest(i, request);
    }
    return new BulkShardRequest(new ShardId("index", "_na_", 0), randomFrom(WriteRequest.RefreshPolicy.values()), items);
}

Example 12 with DocWriteRequest

use of org.opensearch.action.DocWriteRequest in project OpenSearch by opensearch-project.

the class IngestService method executeBulkRequest.

public void executeBulkRequest(int numberOfActionRequests, Iterable<DocWriteRequest<?>> actionRequests, BiConsumer<Integer, Exception> onFailure, BiConsumer<Thread, Exception> onCompletion, IntConsumer onDropped, String executorName) {
    threadPool.executor(executorName).execute(new AbstractRunnable() {

        @Override
        public void onFailure(Exception e) {
            onCompletion.accept(null, e);
        }

        @Override
        protected void doRun() {
            final Thread originalThread = Thread.currentThread();
            final AtomicInteger counter = new AtomicInteger(numberOfActionRequests);
            int i = 0;
            for (DocWriteRequest<?> actionRequest : actionRequests) {
                IndexRequest indexRequest = TransportBulkAction.getIndexWriteRequest(actionRequest);
                if (indexRequest == null) {
                    if (counter.decrementAndGet() == 0) {
                        onCompletion.accept(originalThread, null);
                    }
                    assert counter.get() >= 0;
                    i++;
                    continue;
                }
                final String pipelineId = indexRequest.getPipeline();
                indexRequest.setPipeline(NOOP_PIPELINE_NAME);
                final String finalPipelineId = indexRequest.getFinalPipeline();
                indexRequest.setFinalPipeline(NOOP_PIPELINE_NAME);
                boolean hasFinalPipeline = true;
                final List<String> pipelines;
                if (IngestService.NOOP_PIPELINE_NAME.equals(pipelineId) == false && IngestService.NOOP_PIPELINE_NAME.equals(finalPipelineId) == false) {
                    pipelines = Arrays.asList(pipelineId, finalPipelineId);
                } else if (IngestService.NOOP_PIPELINE_NAME.equals(pipelineId) == false) {
                    pipelines = Collections.singletonList(pipelineId);
                    hasFinalPipeline = false;
                } else if (IngestService.NOOP_PIPELINE_NAME.equals(finalPipelineId) == false) {
                    pipelines = Collections.singletonList(finalPipelineId);
                } else {
                    if (counter.decrementAndGet() == 0) {
                        onCompletion.accept(originalThread, null);
                    }
                    assert counter.get() >= 0;
                    i++;
                    continue;
                }
                executePipelines(i, pipelines.iterator(), hasFinalPipeline, indexRequest, onDropped, onFailure, counter, onCompletion, originalThread);
                i++;
            }
        }
    });
}

Example 13 with DocWriteRequest

use of org.opensearch.action.DocWriteRequest in project OpenSearch by opensearch-project.

the class RequestConverters method bulk.

static Request bulk(BulkRequest bulkRequest) throws IOException {
    Request request = new Request(HttpPost.METHOD_NAME, "/_bulk");
    Params parameters = new Params();
    parameters.withTimeout(bulkRequest.timeout());
    parameters.withRefreshPolicy(bulkRequest.getRefreshPolicy());
    parameters.withPipeline(bulkRequest.pipeline());
    parameters.withRouting(bulkRequest.routing());
    // Bulk API only supports newline delimited JSON or Smile. Before executing
    // the bulk, we need to check that all requests have the same content-type
    // and this content-type is supported by the Bulk API.
    XContentType bulkContentType = null;
    for (int i = 0; i < bulkRequest.numberOfActions(); i++) {
        DocWriteRequest<?> action = bulkRequest.requests().get(i);
        DocWriteRequest.OpType opType = action.opType();
        if (opType == DocWriteRequest.OpType.INDEX || opType == DocWriteRequest.OpType.CREATE) {
            bulkContentType = enforceSameContentType((IndexRequest) action, bulkContentType);
        } else if (opType == DocWriteRequest.OpType.UPDATE) {
            UpdateRequest updateRequest = (UpdateRequest) action;
            if (updateRequest.doc() != null) {
                bulkContentType = enforceSameContentType(updateRequest.doc(), bulkContentType);
            }
            if (updateRequest.upsertRequest() != null) {
                bulkContentType = enforceSameContentType(updateRequest.upsertRequest(), bulkContentType);
            }
        }
    }
    if (bulkContentType == null) {
        bulkContentType = XContentType.JSON;
    }
    final byte separator = bulkContentType.xContent().streamSeparator();
    final ContentType requestContentType = createContentType(bulkContentType);
    ByteArrayOutputStream content = new ByteArrayOutputStream();
    for (DocWriteRequest<?> action : bulkRequest.requests()) {
        DocWriteRequest.OpType opType = action.opType();
        try (XContentBuilder metadata = XContentBuilder.builder(bulkContentType.xContent())) {
            metadata.startObject();
            {
                metadata.startObject(opType.getLowercase());
                if (Strings.hasLength(action.index())) {
                    metadata.field("_index", action.index());
                }
                if (Strings.hasLength(action.id())) {
                    metadata.field("_id", action.id());
                }
                if (Strings.hasLength(action.routing())) {
                    metadata.field("routing", action.routing());
                }
                if (action.version() != Versions.MATCH_ANY) {
                    metadata.field("version", action.version());
                }
                VersionType versionType = action.versionType();
                if (versionType != VersionType.INTERNAL) {
                    if (versionType == VersionType.EXTERNAL) {
                        metadata.field("version_type", "external");
                    } else if (versionType == VersionType.EXTERNAL_GTE) {
                        metadata.field("version_type", "external_gte");
                    }
                }
                if (action.ifSeqNo() != SequenceNumbers.UNASSIGNED_SEQ_NO) {
                    metadata.field("if_seq_no", action.ifSeqNo());
                    metadata.field("if_primary_term", action.ifPrimaryTerm());
                }
                if (opType == DocWriteRequest.OpType.INDEX || opType == DocWriteRequest.OpType.CREATE) {
                    IndexRequest indexRequest = (IndexRequest) action;
                    if (Strings.hasLength(indexRequest.getPipeline())) {
                        metadata.field("pipeline", indexRequest.getPipeline());
                    }
                } else if (opType == DocWriteRequest.OpType.UPDATE) {
                    UpdateRequest updateRequest = (UpdateRequest) action;
                    if (updateRequest.retryOnConflict() > 0) {
                        metadata.field("retry_on_conflict", updateRequest.retryOnConflict());
                    }
                    if (updateRequest.fetchSource() != null) {
                        metadata.field("_source", updateRequest.fetchSource());
                    }
                }
                metadata.endObject();
            }
            metadata.endObject();
            BytesRef metadataSource = BytesReference.bytes(metadata).toBytesRef();
            content.write(metadataSource.bytes, metadataSource.offset, metadataSource.length);
            content.write(separator);
        }
        BytesRef source = null;
        if (opType == DocWriteRequest.OpType.INDEX || opType == DocWriteRequest.OpType.CREATE) {
            IndexRequest indexRequest = (IndexRequest) action;
            BytesReference indexSource = indexRequest.source();
            XContentType indexXContentType = indexRequest.getContentType();
            try (XContentParser parser = XContentHelper.createParser(/*
                         * EMPTY and THROW are fine here because we just call
                         * copyCurrentStructure which doesn't touch the
                         * registry or deprecation.
                         */
            NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, indexSource, indexXContentType)) {
                try (XContentBuilder builder = XContentBuilder.builder(bulkContentType.xContent())) {
                    builder.copyCurrentStructure(parser);
                    source = BytesReference.bytes(builder).toBytesRef();
                }
            }
        } else if (opType == DocWriteRequest.OpType.UPDATE) {
            source = XContentHelper.toXContent((UpdateRequest) action, bulkContentType, false).toBytesRef();
        }
        if (source != null) {
            content.write(source.bytes, source.offset, source.length);
            content.write(separator);
        }
    }
    request.addParameters(parameters.asMap());
    request.setEntity(new NByteArrayEntity(content.toByteArray(), 0, content.size(), requestContentType));
    return request;
}

Example 14 with DocWriteRequest

use of org.opensearch.action.DocWriteRequest in project OpenSearch by opensearch-project.

the class RequestConvertersTests method testBulk.

public void testBulk() throws IOException {
    Map<String, String> expectedParams = new HashMap<>();
    BulkRequest bulkRequest = new BulkRequest();
    if (randomBoolean()) {
        String timeout = randomTimeValue();
        bulkRequest.timeout(timeout);
        expectedParams.put("timeout", timeout);
    } else {
        expectedParams.put("timeout", BulkShardRequest.DEFAULT_TIMEOUT.getStringRep());
    }
    setRandomRefreshPolicy(bulkRequest::setRefreshPolicy, expectedParams);
    XContentType xContentType = randomFrom(XContentType.JSON, XContentType.SMILE);
    int nbItems = randomIntBetween(10, 100);
    DocWriteRequest<?>[] requests = new DocWriteRequest<?>[nbItems];
    for (int i = 0; i < nbItems; i++) {
        String index = randomAlphaOfLength(5);
        String id = randomAlphaOfLength(5);
        BytesReference source = RandomObjects.randomSource(random(), xContentType);
        DocWriteRequest.OpType opType = randomFrom(DocWriteRequest.OpType.values());
        DocWriteRequest<?> docWriteRequest;
        if (opType == DocWriteRequest.OpType.INDEX) {
            IndexRequest indexRequest = new IndexRequest(index).id(id).source(source, xContentType);
            docWriteRequest = indexRequest;
            if (randomBoolean()) {
                indexRequest.setPipeline(randomAlphaOfLength(5));
            }
        } else if (opType == DocWriteRequest.OpType.CREATE) {
            IndexRequest createRequest = new IndexRequest(index).id(id).source(source, xContentType).create(true);
            docWriteRequest = createRequest;
        } else if (opType == DocWriteRequest.OpType.UPDATE) {
            final UpdateRequest updateRequest = new UpdateRequest(index, id).doc(new IndexRequest().source(source, xContentType));
            docWriteRequest = updateRequest;
            if (randomBoolean()) {
                updateRequest.retryOnConflict(randomIntBetween(1, 5));
            }
            if (randomBoolean()) {
                randomizeFetchSourceContextParams(updateRequest::fetchSource, new HashMap<>());
            }
        } else if (opType == DocWriteRequest.OpType.DELETE) {
            docWriteRequest = new DeleteRequest(index, id);
        } else {
            throw new UnsupportedOperationException("optype [" + opType + "] not supported");
        }
        if (randomBoolean()) {
            docWriteRequest.routing(randomAlphaOfLength(10));
        }
        if (opType != DocWriteRequest.OpType.UPDATE && randomBoolean()) {
            docWriteRequest.setIfSeqNo(randomNonNegativeLong());
            docWriteRequest.setIfPrimaryTerm(randomLongBetween(1, 200));
        }
        requests[i] = docWriteRequest;
    }
    bulkRequest.add(requests);
    Request request = RequestConverters.bulk(bulkRequest);
    assertEquals("/_bulk", request.getEndpoint());
    assertEquals(expectedParams, request.getParameters());
    assertEquals(HttpPost.METHOD_NAME, request.getMethod());
    assertEquals(xContentType.mediaTypeWithoutParameters(), request.getEntity().getContentType().getValue());
    byte[] content = new byte[(int) request.getEntity().getContentLength()];
    try (InputStream inputStream = request.getEntity().getContent()) {
        Streams.readFully(inputStream, content);
    }
    BulkRequest parsedBulkRequest = new BulkRequest();
    parsedBulkRequest.add(content, 0, content.length, xContentType);
    assertEquals(bulkRequest.numberOfActions(), parsedBulkRequest.numberOfActions());
    for (int i = 0; i < bulkRequest.numberOfActions(); i++) {
        DocWriteRequest<?> originalRequest = bulkRequest.requests().get(i);
        DocWriteRequest<?> parsedRequest = parsedBulkRequest.requests().get(i);
        assertEquals(originalRequest.opType(), parsedRequest.opType());
        assertEquals(originalRequest.index(), parsedRequest.index());
        assertEquals(originalRequest.id(), parsedRequest.id());
        assertEquals(originalRequest.routing(), parsedRequest.routing());
        assertEquals(originalRequest.version(), parsedRequest.version());
        assertEquals(originalRequest.versionType(), parsedRequest.versionType());
        assertEquals(originalRequest.ifSeqNo(), parsedRequest.ifSeqNo());
        assertEquals(originalRequest.ifPrimaryTerm(), parsedRequest.ifPrimaryTerm());
        DocWriteRequest.OpType opType = originalRequest.opType();
        if (opType == DocWriteRequest.OpType.INDEX) {
            IndexRequest indexRequest = (IndexRequest) originalRequest;
            IndexRequest parsedIndexRequest = (IndexRequest) parsedRequest;
            assertEquals(indexRequest.getPipeline(), parsedIndexRequest.getPipeline());
            assertToXContentEquivalent(indexRequest.source(), parsedIndexRequest.source(), xContentType);
        } else if (opType == DocWriteRequest.OpType.UPDATE) {
            UpdateRequest updateRequest = (UpdateRequest) originalRequest;
            UpdateRequest parsedUpdateRequest = (UpdateRequest) parsedRequest;
            assertEquals(updateRequest.retryOnConflict(), parsedUpdateRequest.retryOnConflict());
            assertEquals(updateRequest.fetchSource(), parsedUpdateRequest.fetchSource());
            if (updateRequest.doc() != null) {
                assertToXContentEquivalent(updateRequest.doc().source(), parsedUpdateRequest.doc().source(), xContentType);
            } else {
                assertNull(parsedUpdateRequest.doc());
            }
        }
    }
}

Example 15 with DocWriteRequest

use of org.opensearch.action.DocWriteRequest in project fesen-httpclient by codelibs.

the class HttpBulkAction method execute.

public void execute(final BulkRequest request, final ActionListener<BulkResponse> listener) {
    // http://ndjson.org/
    final StringBuilder buf = new StringBuilder(10000);
    try {
        final List<DocWriteRequest<?>> bulkRequests = request.requests();
        for (@SuppressWarnings("rawtypes") final DocWriteRequest req : bulkRequests) {
            buf.append(getStringfromDocWriteRequest(req));
            buf.append('\n');
            switch(req.opType().getId()) {
                case 0:
                    {
                        // INDEX
                        buf.append(XContentHelper.convertToJson(((IndexRequest) req).source(), false, XContentType.JSON));
                        buf.append('\n');
                        break;
                    }
                case 1:
                    {
                        // CREATE
                        buf.append(XContentHelper.convertToJson(((IndexRequest) req).source(), false, XContentType.JSON));
                        buf.append('\n');
                        break;
                    }
                case 2:
                    {
                        // UPDATE
                        try (final XContentBuilder builder = ((UpdateRequest) req).toXContent(JsonXContent.contentBuilder(), ToXContent.EMPTY_PARAMS)) {
                            builder.flush();
                            buf.append(BytesReference.bytes(builder).utf8ToString());
                            buf.append('\n');
                        }
                        break;
                    }
                case 3:
                    {
                        // DELETE
                        break;
                    }
                default:
                    break;
            }
        }
    } catch (final IOException e) {
        throw new OpenSearchException("Failed to parse a request.", e);
    }
    getCurlRequest(request).body(buf.toString()).execute(response -> {
        try (final XContentParser parser = createParser(response)) {
            final BulkResponse bulkResponse = fromXContent(parser);
            listener.onResponse(bulkResponse);
        } catch (final Exception e) {
            listener.onFailure(toOpenSearchException(response, e));
        }
    }, e -> unwrapOpenSearchException(listener, e));
}