Examples with BulkRequest org.opensearch.action.bulk.BulkRequest used on opensource projects

Example 1 with BulkRequest

use of org.opensearch.action.bulk.BulkRequest in project OpenSearch by opensearch-project.

the class RequestConverters method bulk.

static Request bulk(BulkRequest bulkRequest) throws IOException {
    Request request = new Request(HttpPost.METHOD_NAME, "/_bulk");
    Params parameters = new Params();
    parameters.withTimeout(bulkRequest.timeout());
    parameters.withRefreshPolicy(bulkRequest.getRefreshPolicy());
    parameters.withPipeline(bulkRequest.pipeline());
    parameters.withRouting(bulkRequest.routing());
    // Bulk API only supports newline delimited JSON or Smile. Before executing
    // the bulk, we need to check that all requests have the same content-type
    // and this content-type is supported by the Bulk API.
    XContentType bulkContentType = null;
    for (int i = 0; i < bulkRequest.numberOfActions(); i++) {
        DocWriteRequest<?> action = bulkRequest.requests().get(i);
        DocWriteRequest.OpType opType = action.opType();
        if (opType == DocWriteRequest.OpType.INDEX || opType == DocWriteRequest.OpType.CREATE) {
            bulkContentType = enforceSameContentType((IndexRequest) action, bulkContentType);
        } else if (opType == DocWriteRequest.OpType.UPDATE) {
            UpdateRequest updateRequest = (UpdateRequest) action;
            if (updateRequest.doc() != null) {
                bulkContentType = enforceSameContentType(updateRequest.doc(), bulkContentType);
            }
            if (updateRequest.upsertRequest() != null) {
                bulkContentType = enforceSameContentType(updateRequest.upsertRequest(), bulkContentType);
            }
        }
    }
    if (bulkContentType == null) {
        bulkContentType = XContentType.JSON;
    }
    final byte separator = bulkContentType.xContent().streamSeparator();
    final ContentType requestContentType = createContentType(bulkContentType);
    ByteArrayOutputStream content = new ByteArrayOutputStream();
    for (DocWriteRequest<?> action : bulkRequest.requests()) {
        DocWriteRequest.OpType opType = action.opType();
        try (XContentBuilder metadata = XContentBuilder.builder(bulkContentType.xContent())) {
            metadata.startObject();
            {
                metadata.startObject(opType.getLowercase());
                if (Strings.hasLength(action.index())) {
                    metadata.field("_index", action.index());
                }
                if (Strings.hasLength(action.id())) {
                    metadata.field("_id", action.id());
                }
                if (Strings.hasLength(action.routing())) {
                    metadata.field("routing", action.routing());
                }
                if (action.version() != Versions.MATCH_ANY) {
                    metadata.field("version", action.version());
                }
                VersionType versionType = action.versionType();
                if (versionType != VersionType.INTERNAL) {
                    if (versionType == VersionType.EXTERNAL) {
                        metadata.field("version_type", "external");
                    } else if (versionType == VersionType.EXTERNAL_GTE) {
                        metadata.field("version_type", "external_gte");
                    }
                }
                if (action.ifSeqNo() != SequenceNumbers.UNASSIGNED_SEQ_NO) {
                    metadata.field("if_seq_no", action.ifSeqNo());
                    metadata.field("if_primary_term", action.ifPrimaryTerm());
                }
                if (opType == DocWriteRequest.OpType.INDEX || opType == DocWriteRequest.OpType.CREATE) {
                    IndexRequest indexRequest = (IndexRequest) action;
                    if (Strings.hasLength(indexRequest.getPipeline())) {
                        metadata.field("pipeline", indexRequest.getPipeline());
                    }
                } else if (opType == DocWriteRequest.OpType.UPDATE) {
                    UpdateRequest updateRequest = (UpdateRequest) action;
                    if (updateRequest.retryOnConflict() > 0) {
                        metadata.field("retry_on_conflict", updateRequest.retryOnConflict());
                    }
                    if (updateRequest.fetchSource() != null) {
                        metadata.field("_source", updateRequest.fetchSource());
                    }
                }
                metadata.endObject();
            }
            metadata.endObject();
            BytesRef metadataSource = BytesReference.bytes(metadata).toBytesRef();
            content.write(metadataSource.bytes, metadataSource.offset, metadataSource.length);
            content.write(separator);
        }
        BytesRef source = null;
        if (opType == DocWriteRequest.OpType.INDEX || opType == DocWriteRequest.OpType.CREATE) {
            IndexRequest indexRequest = (IndexRequest) action;
            BytesReference indexSource = indexRequest.source();
            XContentType indexXContentType = indexRequest.getContentType();
            try (XContentParser parser = XContentHelper.createParser(/*
                         * EMPTY and THROW are fine here because we just call
                         * copyCurrentStructure which doesn't touch the
                         * registry or deprecation.
                         */
            NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, indexSource, indexXContentType)) {
                try (XContentBuilder builder = XContentBuilder.builder(bulkContentType.xContent())) {
                    builder.copyCurrentStructure(parser);
                    source = BytesReference.bytes(builder).toBytesRef();
                }
            }
        } else if (opType == DocWriteRequest.OpType.UPDATE) {
            source = XContentHelper.toXContent((UpdateRequest) action, bulkContentType, false).toBytesRef();
        }
        if (source != null) {
            content.write(source.bytes, source.offset, source.length);
            content.write(separator);
        }
    }
    request.addParameters(parameters.asMap());
    request.setEntity(new NByteArrayEntity(content.toByteArray(), 0, content.size(), requestContentType));
    return request;
}

Example 2 with BulkRequest

use of org.opensearch.action.bulk.BulkRequest in project OpenSearch by opensearch-project.

the class BulkRequestWithGlobalParametersIT method testIndexGlobalAndPerRequest.

@SuppressWarnings("unchecked")
public void testIndexGlobalAndPerRequest() throws IOException {
    BulkRequest request = new BulkRequest("global_index");
    request.add(new IndexRequest("local_index").id("1").source(XContentType.JSON, "field", "bulk1"));
    request.add(// will take global index
    new IndexRequest().id("2").source(XContentType.JSON, "field", "bulk2"));
    bulk(request);
    Iterable<SearchHit> hits = searchAll("local_index", "global_index");
    assertThat(hits, containsInAnyOrder(both(hasId("1")).and(hasIndex("local_index")), both(hasId("2")).and(hasIndex("global_index"))));
}

Example 3 with BulkRequest

use of org.opensearch.action.bulk.BulkRequest in project OpenSearch by opensearch-project.

the class BulkRequestWithGlobalParametersIT method testGlobalIndex.

public void testGlobalIndex() throws IOException {
    BulkRequest request = new BulkRequest("global_index");
    request.add(new IndexRequest().id("1").source(XContentType.JSON, "field", "bulk1"));
    request.add(new IndexRequest().id("2").source(XContentType.JSON, "field", "bulk2"));
    bulk(request);
    Iterable<SearchHit> hits = searchAll("global_index");
    assertThat(hits, everyItem(hasIndex("global_index")));
}

Example 4 with BulkRequest

use of org.opensearch.action.bulk.BulkRequest in project OpenSearch by opensearch-project.

the class BulkRequestWithGlobalParametersIT method testGlobalPipelineOnBulkRequest.

public void testGlobalPipelineOnBulkRequest() throws IOException {
    createFieldAddingPipleine("xyz", "fieldNameXYZ", "valueXYZ");
    BulkRequest request = new BulkRequest();
    request.add(new IndexRequest("test").id("1").source(XContentType.JSON, "field", "bulk1"));
    request.add(new IndexRequest("test").id("2").source(XContentType.JSON, "field", "bulk2"));
    request.pipeline("xyz");
    bulk(request);
    Iterable<SearchHit> hits = searchAll("test");
    assertThat(hits, containsInAnyOrder(hasId("1"), hasId("2")));
    assertThat(hits, everyItem(hasProperty(fieldFromSource("fieldNameXYZ"), equalTo("valueXYZ"))));
}

Example 5 with BulkRequest

use of org.opensearch.action.bulk.BulkRequest in project OpenSearch by opensearch-project.

the class BulkRequestWithGlobalParametersIT method testGlobalRouting.

public void testGlobalRouting() throws IOException {
    createIndexWithMultipleShards("index");
    BulkRequest request = new BulkRequest((String) null);
    request.add(new IndexRequest("index").id("1").source(XContentType.JSON, "field", "bulk1"));
    request.add(new IndexRequest("index").id("2").source(XContentType.JSON, "field", "bulk1"));
    request.routing("1");
    bulk(request);
    Iterable<SearchHit> emptyHits = searchAll(new SearchRequest("index").routing("xxx"));
    assertThat(emptyHits, is(emptyIterable()));
    Iterable<SearchHit> hits = searchAll(new SearchRequest("index").routing("1"));
    assertThat(hits, containsInAnyOrder(hasId("1"), hasId("2")));
}