Examples with BytesReference org.opensearch.common.bytes.BytesReference used on opensource projects

Example 1 with BytesReference

use of org.opensearch.common.bytes.BytesReference in project OpenSearch by opensearch-project.

the class RequestConverters method bulk.

static Request bulk(BulkRequest bulkRequest) throws IOException {
    Request request = new Request(HttpPost.METHOD_NAME, "/_bulk");
    Params parameters = new Params();
    parameters.withTimeout(bulkRequest.timeout());
    parameters.withRefreshPolicy(bulkRequest.getRefreshPolicy());
    parameters.withPipeline(bulkRequest.pipeline());
    parameters.withRouting(bulkRequest.routing());
    // Bulk API only supports newline delimited JSON or Smile. Before executing
    // the bulk, we need to check that all requests have the same content-type
    // and this content-type is supported by the Bulk API.
    XContentType bulkContentType = null;
    for (int i = 0; i < bulkRequest.numberOfActions(); i++) {
        DocWriteRequest<?> action = bulkRequest.requests().get(i);
        DocWriteRequest.OpType opType = action.opType();
        if (opType == DocWriteRequest.OpType.INDEX || opType == DocWriteRequest.OpType.CREATE) {
            bulkContentType = enforceSameContentType((IndexRequest) action, bulkContentType);
        } else if (opType == DocWriteRequest.OpType.UPDATE) {
            UpdateRequest updateRequest = (UpdateRequest) action;
            if (updateRequest.doc() != null) {
                bulkContentType = enforceSameContentType(updateRequest.doc(), bulkContentType);
            }
            if (updateRequest.upsertRequest() != null) {
                bulkContentType = enforceSameContentType(updateRequest.upsertRequest(), bulkContentType);
            }
        }
    }
    if (bulkContentType == null) {
        bulkContentType = XContentType.JSON;
    }
    final byte separator = bulkContentType.xContent().streamSeparator();
    final ContentType requestContentType = createContentType(bulkContentType);
    ByteArrayOutputStream content = new ByteArrayOutputStream();
    for (DocWriteRequest<?> action : bulkRequest.requests()) {
        DocWriteRequest.OpType opType = action.opType();
        try (XContentBuilder metadata = XContentBuilder.builder(bulkContentType.xContent())) {
            metadata.startObject();
            {
                metadata.startObject(opType.getLowercase());
                if (Strings.hasLength(action.index())) {
                    metadata.field("_index", action.index());
                }
                if (Strings.hasLength(action.id())) {
                    metadata.field("_id", action.id());
                }
                if (Strings.hasLength(action.routing())) {
                    metadata.field("routing", action.routing());
                }
                if (action.version() != Versions.MATCH_ANY) {
                    metadata.field("version", action.version());
                }
                VersionType versionType = action.versionType();
                if (versionType != VersionType.INTERNAL) {
                    if (versionType == VersionType.EXTERNAL) {
                        metadata.field("version_type", "external");
                    } else if (versionType == VersionType.EXTERNAL_GTE) {
                        metadata.field("version_type", "external_gte");
                    }
                }
                if (action.ifSeqNo() != SequenceNumbers.UNASSIGNED_SEQ_NO) {
                    metadata.field("if_seq_no", action.ifSeqNo());
                    metadata.field("if_primary_term", action.ifPrimaryTerm());
                }
                if (opType == DocWriteRequest.OpType.INDEX || opType == DocWriteRequest.OpType.CREATE) {
                    IndexRequest indexRequest = (IndexRequest) action;
                    if (Strings.hasLength(indexRequest.getPipeline())) {
                        metadata.field("pipeline", indexRequest.getPipeline());
                    }
                } else if (opType == DocWriteRequest.OpType.UPDATE) {
                    UpdateRequest updateRequest = (UpdateRequest) action;
                    if (updateRequest.retryOnConflict() > 0) {
                        metadata.field("retry_on_conflict", updateRequest.retryOnConflict());
                    }
                    if (updateRequest.fetchSource() != null) {
                        metadata.field("_source", updateRequest.fetchSource());
                    }
                }
                metadata.endObject();
            }
            metadata.endObject();
            BytesRef metadataSource = BytesReference.bytes(metadata).toBytesRef();
            content.write(metadataSource.bytes, metadataSource.offset, metadataSource.length);
            content.write(separator);
        }
        BytesRef source = null;
        if (opType == DocWriteRequest.OpType.INDEX || opType == DocWriteRequest.OpType.CREATE) {
            IndexRequest indexRequest = (IndexRequest) action;
            BytesReference indexSource = indexRequest.source();
            XContentType indexXContentType = indexRequest.getContentType();
            try (XContentParser parser = XContentHelper.createParser(/*
                         * EMPTY and THROW are fine here because we just call
                         * copyCurrentStructure which doesn't touch the
                         * registry or deprecation.
                         */
            NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, indexSource, indexXContentType)) {
                try (XContentBuilder builder = XContentBuilder.builder(bulkContentType.xContent())) {
                    builder.copyCurrentStructure(parser);
                    source = BytesReference.bytes(builder).toBytesRef();
                }
            }
        } else if (opType == DocWriteRequest.OpType.UPDATE) {
            source = XContentHelper.toXContent((UpdateRequest) action, bulkContentType, false).toBytesRef();
        }
        if (source != null) {
            content.write(source.bytes, source.offset, source.length);
            content.write(separator);
        }
    }
    request.addParameters(parameters.asMap());
    request.setEntity(new NByteArrayEntity(content.toByteArray(), 0, content.size(), requestContentType));
    return request;
}

Example 2 with BytesReference

use of org.opensearch.common.bytes.BytesReference in project OpenSearch by opensearch-project.

the class TermVectorsRequest method toXContent.

@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
    builder.startObject();
    builder.field("_index", index);
    if (type != null) {
        builder.field("_type", type);
    }
    if (id != null)
        builder.field("_id", id);
    if (fields != null)
        builder.field("fields", fields);
    // set values only when different from defaults
    if (requestPositions == false)
        builder.field("positions", false);
    if (requestPayloads == false)
        builder.field("payloads", false);
    if (requestOffsets == false)
        builder.field("offsets", false);
    if (requestFieldStatistics == false)
        builder.field("field_statistics", false);
    if (requestTermStatistics)
        builder.field("term_statistics", true);
    if (perFieldAnalyzer != null)
        builder.field("per_field_analyzer", perFieldAnalyzer);
    if (docBuilder != null) {
        BytesReference doc = BytesReference.bytes(docBuilder);
        try (InputStream stream = doc.streamInput()) {
            builder.rawField("doc", stream, docBuilder.contentType());
        }
    }
    if (filterSettings != null) {
        builder.startObject("filter");
        String[] filterSettingNames = { "max_num_terms", "min_term_freq", "max_term_freq", "min_doc_freq", "max_doc_freq", "min_word_length", "max_word_length" };
        for (String settingName : filterSettingNames) {
            if (filterSettings.containsKey(settingName))
                builder.field(settingName, filterSettings.get(settingName));
        }
        builder.endObject();
    }
    builder.endObject();
    return builder;
}

Example 3 with BytesReference

use of org.opensearch.common.bytes.BytesReference in project OpenSearch by opensearch-project.

the class RequestConvertersTests method testUpdate.

public void testUpdate() throws IOException {
    XContentType xContentType = randomFrom(XContentType.values());
    Map<String, String> expectedParams = new HashMap<>();
    String index = randomAlphaOfLengthBetween(3, 10);
    String id = randomAlphaOfLengthBetween(3, 10);
    UpdateRequest updateRequest = new UpdateRequest(index, id);
    updateRequest.detectNoop(randomBoolean());
    if (randomBoolean()) {
        BytesReference source = RandomObjects.randomSource(random(), xContentType);
        updateRequest.doc(new IndexRequest().source(source, xContentType));
        boolean docAsUpsert = randomBoolean();
        updateRequest.docAsUpsert(docAsUpsert);
        if (docAsUpsert) {
            expectedParams.put("doc_as_upsert", "true");
        }
    } else {
        updateRequest.script(mockScript("_value + 1"));
        updateRequest.scriptedUpsert(randomBoolean());
    }
    if (randomBoolean()) {
        BytesReference source = RandomObjects.randomSource(random(), xContentType);
        updateRequest.upsert(new IndexRequest().source(source, xContentType));
    }
    if (randomBoolean()) {
        String routing = randomAlphaOfLengthBetween(3, 10);
        updateRequest.routing(routing);
        expectedParams.put("routing", routing);
    }
    if (randomBoolean()) {
        String timeout = randomTimeValue();
        updateRequest.timeout(timeout);
        expectedParams.put("timeout", timeout);
    } else {
        expectedParams.put("timeout", ReplicationRequest.DEFAULT_TIMEOUT.getStringRep());
    }
    if (randomBoolean()) {
        WriteRequest.RefreshPolicy refreshPolicy = randomFrom(WriteRequest.RefreshPolicy.values());
        updateRequest.setRefreshPolicy(refreshPolicy);
        if (refreshPolicy != WriteRequest.RefreshPolicy.NONE) {
            expectedParams.put("refresh", refreshPolicy.getValue());
        }
    }
    setRandomWaitForActiveShards(updateRequest::waitForActiveShards, expectedParams);
    // if* params are passed in the body
    setRandomIfSeqNoAndTerm(updateRequest, new HashMap<>());
    if (randomBoolean()) {
        int retryOnConflict = randomIntBetween(0, 5);
        updateRequest.retryOnConflict(retryOnConflict);
        if (retryOnConflict > 0) {
            expectedParams.put("retry_on_conflict", String.valueOf(retryOnConflict));
        }
    }
    if (randomBoolean()) {
        randomizeFetchSourceContextParams(updateRequest::fetchSource, expectedParams);
    }
    Request request = RequestConverters.update(updateRequest);
    assertEquals("/" + index + "/_update/" + id, request.getEndpoint());
    assertEquals(expectedParams, request.getParameters());
    assertEquals(HttpPost.METHOD_NAME, request.getMethod());
    HttpEntity entity = request.getEntity();
    assertTrue(entity instanceof NByteArrayEntity);
    UpdateRequest parsedUpdateRequest = new UpdateRequest();
    XContentType entityContentType = XContentType.fromMediaTypeOrFormat(entity.getContentType().getValue());
    try (XContentParser parser = createParser(entityContentType.xContent(), entity.getContent())) {
        parsedUpdateRequest.fromXContent(parser);
    }
    assertEquals(updateRequest.scriptedUpsert(), parsedUpdateRequest.scriptedUpsert());
    assertEquals(updateRequest.docAsUpsert(), parsedUpdateRequest.docAsUpsert());
    assertEquals(updateRequest.detectNoop(), parsedUpdateRequest.detectNoop());
    assertEquals(updateRequest.fetchSource(), parsedUpdateRequest.fetchSource());
    assertIfSeqNoAndTerm(updateRequest, parsedUpdateRequest);
    assertEquals(updateRequest.script(), parsedUpdateRequest.script());
    if (updateRequest.doc() != null) {
        assertToXContentEquivalent(updateRequest.doc().source(), parsedUpdateRequest.doc().source(), xContentType);
    } else {
        assertNull(parsedUpdateRequest.doc());
    }
    if (updateRequest.upsertRequest() != null) {
        assertToXContentEquivalent(updateRequest.upsertRequest().source(), parsedUpdateRequest.upsertRequest().source(), xContentType);
    } else {
        assertNull(parsedUpdateRequest.upsertRequest());
    }
}

Example 4 with BytesReference

use of org.opensearch.common.bytes.BytesReference in project OpenSearch by opensearch-project.

the class RequestConvertersTests method testBulk.

public void testBulk() throws IOException {
    Map<String, String> expectedParams = new HashMap<>();
    BulkRequest bulkRequest = new BulkRequest();
    if (randomBoolean()) {
        String timeout = randomTimeValue();
        bulkRequest.timeout(timeout);
        expectedParams.put("timeout", timeout);
    } else {
        expectedParams.put("timeout", BulkShardRequest.DEFAULT_TIMEOUT.getStringRep());
    }
    setRandomRefreshPolicy(bulkRequest::setRefreshPolicy, expectedParams);
    XContentType xContentType = randomFrom(XContentType.JSON, XContentType.SMILE);
    int nbItems = randomIntBetween(10, 100);
    DocWriteRequest<?>[] requests = new DocWriteRequest<?>[nbItems];
    for (int i = 0; i < nbItems; i++) {
        String index = randomAlphaOfLength(5);
        String id = randomAlphaOfLength(5);
        BytesReference source = RandomObjects.randomSource(random(), xContentType);
        DocWriteRequest.OpType opType = randomFrom(DocWriteRequest.OpType.values());
        DocWriteRequest<?> docWriteRequest;
        if (opType == DocWriteRequest.OpType.INDEX) {
            IndexRequest indexRequest = new IndexRequest(index).id(id).source(source, xContentType);
            docWriteRequest = indexRequest;
            if (randomBoolean()) {
                indexRequest.setPipeline(randomAlphaOfLength(5));
            }
        } else if (opType == DocWriteRequest.OpType.CREATE) {
            IndexRequest createRequest = new IndexRequest(index).id(id).source(source, xContentType).create(true);
            docWriteRequest = createRequest;
        } else if (opType == DocWriteRequest.OpType.UPDATE) {
            final UpdateRequest updateRequest = new UpdateRequest(index, id).doc(new IndexRequest().source(source, xContentType));
            docWriteRequest = updateRequest;
            if (randomBoolean()) {
                updateRequest.retryOnConflict(randomIntBetween(1, 5));
            }
            if (randomBoolean()) {
                randomizeFetchSourceContextParams(updateRequest::fetchSource, new HashMap<>());
            }
        } else if (opType == DocWriteRequest.OpType.DELETE) {
            docWriteRequest = new DeleteRequest(index, id);
        } else {
            throw new UnsupportedOperationException("optype [" + opType + "] not supported");
        }
        if (randomBoolean()) {
            docWriteRequest.routing(randomAlphaOfLength(10));
        }
        if (opType != DocWriteRequest.OpType.UPDATE && randomBoolean()) {
            docWriteRequest.setIfSeqNo(randomNonNegativeLong());
            docWriteRequest.setIfPrimaryTerm(randomLongBetween(1, 200));
        }
        requests[i] = docWriteRequest;
    }
    bulkRequest.add(requests);
    Request request = RequestConverters.bulk(bulkRequest);
    assertEquals("/_bulk", request.getEndpoint());
    assertEquals(expectedParams, request.getParameters());
    assertEquals(HttpPost.METHOD_NAME, request.getMethod());
    assertEquals(xContentType.mediaTypeWithoutParameters(), request.getEntity().getContentType().getValue());
    byte[] content = new byte[(int) request.getEntity().getContentLength()];
    try (InputStream inputStream = request.getEntity().getContent()) {
        Streams.readFully(inputStream, content);
    }
    BulkRequest parsedBulkRequest = new BulkRequest();
    parsedBulkRequest.add(content, 0, content.length, xContentType);
    assertEquals(bulkRequest.numberOfActions(), parsedBulkRequest.numberOfActions());
    for (int i = 0; i < bulkRequest.numberOfActions(); i++) {
        DocWriteRequest<?> originalRequest = bulkRequest.requests().get(i);
        DocWriteRequest<?> parsedRequest = parsedBulkRequest.requests().get(i);
        assertEquals(originalRequest.opType(), parsedRequest.opType());
        assertEquals(originalRequest.index(), parsedRequest.index());
        assertEquals(originalRequest.id(), parsedRequest.id());
        assertEquals(originalRequest.routing(), parsedRequest.routing());
        assertEquals(originalRequest.version(), parsedRequest.version());
        assertEquals(originalRequest.versionType(), parsedRequest.versionType());
        assertEquals(originalRequest.ifSeqNo(), parsedRequest.ifSeqNo());
        assertEquals(originalRequest.ifPrimaryTerm(), parsedRequest.ifPrimaryTerm());
        DocWriteRequest.OpType opType = originalRequest.opType();
        if (opType == DocWriteRequest.OpType.INDEX) {
            IndexRequest indexRequest = (IndexRequest) originalRequest;
            IndexRequest parsedIndexRequest = (IndexRequest) parsedRequest;
            assertEquals(indexRequest.getPipeline(), parsedIndexRequest.getPipeline());
            assertToXContentEquivalent(indexRequest.source(), parsedIndexRequest.source(), xContentType);
        } else if (opType == DocWriteRequest.OpType.UPDATE) {
            UpdateRequest updateRequest = (UpdateRequest) originalRequest;
            UpdateRequest parsedUpdateRequest = (UpdateRequest) parsedRequest;
            assertEquals(updateRequest.retryOnConflict(), parsedUpdateRequest.retryOnConflict());
            assertEquals(updateRequest.fetchSource(), parsedUpdateRequest.fetchSource());
            if (updateRequest.doc() != null) {
                assertToXContentEquivalent(updateRequest.doc().source(), parsedUpdateRequest.doc().source(), xContentType);
            } else {
                assertNull(parsedUpdateRequest.doc());
            }
        }
    }
}

Example 5 with BytesReference

use of org.opensearch.common.bytes.BytesReference in project OpenSearch by opensearch-project.

the class RequestConvertersTests method assertToXContentBody.

static void assertToXContentBody(ToXContent expectedBody, HttpEntity actualEntity) throws IOException {
    BytesReference expectedBytes = XContentHelper.toXContent(expectedBody, REQUEST_BODY_CONTENT_TYPE, false);
    assertEquals(XContentType.JSON.mediaTypeWithoutParameters(), actualEntity.getContentType().getValue());
    assertEquals(expectedBytes, new BytesArray(EntityUtils.toByteArray(actualEntity)));
}