Examples with IndexRequest org.elasticsearch.action.index.IndexRequest used on opensource projects

Example 41 with IndexRequest

use of org.elasticsearch.action.index.IndexRequest in project elasticsearch by elastic.

the class RequestTests method testBulk.

public void testBulk() throws IOException {
    Map<String, String> expectedParams = new HashMap<>();
    BulkRequest bulkRequest = new BulkRequest();
    if (randomBoolean()) {
        String timeout = randomTimeValue();
        bulkRequest.timeout(timeout);
        expectedParams.put("timeout", timeout);
    } else {
        expectedParams.put("timeout", BulkShardRequest.DEFAULT_TIMEOUT.getStringRep());
    }
    if (randomBoolean()) {
        WriteRequest.RefreshPolicy refreshPolicy = randomFrom(WriteRequest.RefreshPolicy.values());
        bulkRequest.setRefreshPolicy(refreshPolicy);
        if (refreshPolicy != WriteRequest.RefreshPolicy.NONE) {
            expectedParams.put("refresh", refreshPolicy.getValue());
        }
    }
    XContentType xContentType = randomFrom(XContentType.JSON, XContentType.SMILE);
    int nbItems = randomIntBetween(10, 100);
    for (int i = 0; i < nbItems; i++) {
        String index = randomAsciiOfLength(5);
        String type = randomAsciiOfLength(5);
        String id = randomAsciiOfLength(5);
        BytesReference source = RandomObjects.randomSource(random(), xContentType);
        DocWriteRequest.OpType opType = randomFrom(DocWriteRequest.OpType.values());
        DocWriteRequest<?> docWriteRequest = null;
        if (opType == DocWriteRequest.OpType.INDEX) {
            IndexRequest indexRequest = new IndexRequest(index, type, id).source(source, xContentType);
            docWriteRequest = indexRequest;
            if (randomBoolean()) {
                indexRequest.setPipeline(randomAsciiOfLength(5));
            }
            if (randomBoolean()) {
                indexRequest.parent(randomAsciiOfLength(5));
            }
        } else if (opType == DocWriteRequest.OpType.CREATE) {
            IndexRequest createRequest = new IndexRequest(index, type, id).source(source, xContentType).create(true);
            docWriteRequest = createRequest;
            if (randomBoolean()) {
                createRequest.parent(randomAsciiOfLength(5));
            }
        } else if (opType == DocWriteRequest.OpType.UPDATE) {
            final UpdateRequest updateRequest = new UpdateRequest(index, type, id).doc(new IndexRequest().source(source, xContentType));
            docWriteRequest = updateRequest;
            if (randomBoolean()) {
                updateRequest.retryOnConflict(randomIntBetween(1, 5));
            }
            if (randomBoolean()) {
                randomizeFetchSourceContextParams(updateRequest::fetchSource, new HashMap<>());
            }
            if (randomBoolean()) {
                updateRequest.parent(randomAsciiOfLength(5));
            }
        } else if (opType == DocWriteRequest.OpType.DELETE) {
            docWriteRequest = new DeleteRequest(index, type, id);
        }
        if (randomBoolean()) {
            docWriteRequest.routing(randomAsciiOfLength(10));
        }
        if (randomBoolean()) {
            docWriteRequest.version(randomNonNegativeLong());
        }
        if (randomBoolean()) {
            docWriteRequest.versionType(randomFrom(VersionType.values()));
        }
        bulkRequest.add(docWriteRequest);
    }
    Request request = Request.bulk(bulkRequest);
    assertEquals("/_bulk", request.endpoint);
    assertEquals(expectedParams, request.params);
    assertEquals("POST", request.method);
    byte[] content = new byte[(int) request.entity.getContentLength()];
    try (InputStream inputStream = request.entity.getContent()) {
        Streams.readFully(inputStream, content);
    }
    BulkRequest parsedBulkRequest = new BulkRequest();
    parsedBulkRequest.add(content, 0, content.length, xContentType);
    assertEquals(bulkRequest.numberOfActions(), parsedBulkRequest.numberOfActions());
    for (int i = 0; i < bulkRequest.numberOfActions(); i++) {
        DocWriteRequest<?> originalRequest = bulkRequest.requests().get(i);
        DocWriteRequest<?> parsedRequest = parsedBulkRequest.requests().get(i);
        assertEquals(originalRequest.opType(), parsedRequest.opType());
        assertEquals(originalRequest.index(), parsedRequest.index());
        assertEquals(originalRequest.type(), parsedRequest.type());
        assertEquals(originalRequest.id(), parsedRequest.id());
        assertEquals(originalRequest.routing(), parsedRequest.routing());
        assertEquals(originalRequest.parent(), parsedRequest.parent());
        assertEquals(originalRequest.version(), parsedRequest.version());
        assertEquals(originalRequest.versionType(), parsedRequest.versionType());
        DocWriteRequest.OpType opType = originalRequest.opType();
        if (opType == DocWriteRequest.OpType.INDEX) {
            IndexRequest indexRequest = (IndexRequest) originalRequest;
            IndexRequest parsedIndexRequest = (IndexRequest) parsedRequest;
            assertEquals(indexRequest.getPipeline(), parsedIndexRequest.getPipeline());
            assertToXContentEquivalent(indexRequest.source(), parsedIndexRequest.source(), xContentType);
        } else if (opType == DocWriteRequest.OpType.UPDATE) {
            UpdateRequest updateRequest = (UpdateRequest) originalRequest;
            UpdateRequest parsedUpdateRequest = (UpdateRequest) parsedRequest;
            assertEquals(updateRequest.retryOnConflict(), parsedUpdateRequest.retryOnConflict());
            assertEquals(updateRequest.fetchSource(), parsedUpdateRequest.fetchSource());
            if (updateRequest.doc() != null) {
                assertToXContentEquivalent(updateRequest.doc().source(), parsedUpdateRequest.doc().source(), xContentType);
            } else {
                assertNull(parsedUpdateRequest.doc());
            }
        }
    }
}

Example 42 with IndexRequest

use of org.elasticsearch.action.index.IndexRequest in project elasticsearch by elastic.

the class RequestTests method testIndex.

public void testIndex() throws IOException {
    String index = randomAsciiOfLengthBetween(3, 10);
    String type = randomAsciiOfLengthBetween(3, 10);
    IndexRequest indexRequest = new IndexRequest(index, type);
    String id = randomBoolean() ? randomAsciiOfLengthBetween(3, 10) : null;
    indexRequest.id(id);
    Map<String, String> expectedParams = new HashMap<>();
    String method = "POST";
    if (id != null) {
        method = "PUT";
        if (randomBoolean()) {
            indexRequest.opType(DocWriteRequest.OpType.CREATE);
        }
    }
    setRandomTimeout(indexRequest, expectedParams);
    setRandomRefreshPolicy(indexRequest, expectedParams);
    // There is some logic around _create endpoint and version/version type
    if (indexRequest.opType() == DocWriteRequest.OpType.CREATE) {
        indexRequest.version(randomFrom(Versions.MATCH_ANY, Versions.MATCH_DELETED));
        expectedParams.put("version", Long.toString(Versions.MATCH_DELETED));
    } else {
        setRandomVersion(indexRequest, expectedParams);
        setRandomVersionType(indexRequest, expectedParams);
    }
    if (frequently()) {
        if (randomBoolean()) {
            String routing = randomAsciiOfLengthBetween(3, 10);
            indexRequest.routing(routing);
            expectedParams.put("routing", routing);
        }
        if (randomBoolean()) {
            String parent = randomAsciiOfLengthBetween(3, 10);
            indexRequest.parent(parent);
            expectedParams.put("parent", parent);
        }
        if (randomBoolean()) {
            String pipeline = randomAsciiOfLengthBetween(3, 10);
            indexRequest.setPipeline(pipeline);
            expectedParams.put("pipeline", pipeline);
        }
    }
    XContentType xContentType = randomFrom(XContentType.values());
    int nbFields = randomIntBetween(0, 10);
    try (XContentBuilder builder = XContentBuilder.builder(xContentType.xContent())) {
        builder.startObject();
        for (int i = 0; i < nbFields; i++) {
            builder.field("field_" + i, i);
        }
        builder.endObject();
        indexRequest.source(builder);
    }
    Request request = Request.index(indexRequest);
    if (indexRequest.opType() == DocWriteRequest.OpType.CREATE) {
        assertEquals("/" + index + "/" + type + "/" + id + "/_create", request.endpoint);
    } else if (id != null) {
        assertEquals("/" + index + "/" + type + "/" + id, request.endpoint);
    } else {
        assertEquals("/" + index + "/" + type, request.endpoint);
    }
    assertEquals(expectedParams, request.params);
    assertEquals(method, request.method);
    HttpEntity entity = request.entity;
    assertNotNull(entity);
    assertTrue(entity instanceof ByteArrayEntity);
    try (XContentParser parser = createParser(xContentType.xContent(), entity.getContent())) {
        assertEquals(nbFields, parser.map().size());
    }
}

Example 43 with IndexRequest

use of org.elasticsearch.action.index.IndexRequest in project wonderdog by infochimps-labs.

the class ElasticSearchStreamingRecordWriter method index.

private void index(String json) throws IOException {
    Map<String, Object> record = mapper.readValue(json, Map.class);
    IndexRequest request = null;
    if (record.containsKey(idFieldName)) {
        Object idValue = record.get(idFieldName);
        request = Requests.indexRequest(indexNameForRecord(record)).id(String.valueOf(idValue)).type(mappingNameForRecord(record)).create(false).source(json);
    } else {
        request = Requests.indexRequest(indexNameForRecord(record)).type(mappingNameForRecord(record)).source(json);
    }
    if (record.containsKey(routingFieldName)) {
        Object routingValue = record.get(routingFieldName);
        request.routing(String.valueOf(routingValue));
    }
    currentRequest.add(request);
}

Example 44 with IndexRequest

use of org.elasticsearch.action.index.IndexRequest in project sonarqube by SonarSource.

the class ProxyBulkRequestBuilder method toString.

@Override
public String toString() {
    StringBuilder message = new StringBuilder();
    message.append("Bulk[");
    Multiset<BulkRequestKey> groupedRequests = LinkedHashMultiset.create();
    for (int i = 0; i < request.requests().size(); i++) {
        ActionRequest<?> item = request.requests().get(i);
        String requestType;
        String index;
        String docType;
        if (item instanceof IndexRequest) {
            IndexRequest request = (IndexRequest) item;
            requestType = "index";
            index = request.index();
            docType = request.type();
        } else if (item instanceof UpdateRequest) {
            UpdateRequest request = (UpdateRequest) item;
            requestType = "update";
            index = request.index();
            docType = request.type();
        } else if (item instanceof DeleteRequest) {
            DeleteRequest request = (DeleteRequest) item;
            requestType = "delete";
            index = request.index();
            docType = request.type();
        } else {
            // Cannot happen, not allowed by BulkRequest's contract
            throw new IllegalStateException("Unsupported bulk request type: " + item.getClass());
        }
        groupedRequests.add(new BulkRequestKey(requestType, index, docType));
    }
    Set<Multiset.Entry<BulkRequestKey>> entrySet = groupedRequests.entrySet();
    int size = entrySet.size();
    int current = 0;
    for (Multiset.Entry<BulkRequestKey> requestEntry : entrySet) {
        message.append(requestEntry.getCount()).append(" ").append(requestEntry.getElement().toString());
        current++;
        if (current < size) {
            message.append(", ");
        }
    }
    message.append("]");
    return message.toString();
}

Example 45 with IndexRequest

use of org.elasticsearch.action.index.IndexRequest in project jena by apache.

the class TextIndexES method addEntity.

/**
     * Add an Entity to the ElasticSearch Index.
     * The entity will be added as a new document in ES, if it does not already exists.
     * If the Entity exists, then the entity will simply be updated.
     * The entity will never be replaced.
     * @param entity the entity to add
     */
@Override
public void addEntity(Entity entity) {
    LOGGER.debug("Adding/Updating the entity in ES");
    //The field that has a not null value in the current Entity instance.
    //Required, mainly for building a script for the update command.
    String fieldToAdd = null;
    String fieldValueToAdd = null;
    try {
        XContentBuilder builder = jsonBuilder().startObject();
        for (String field : docDef.fields()) {
            if (entity.get(field) != null) {
                if (entity.getLanguage() != null && !entity.getLanguage().isEmpty()) {
                    //We make sure that the field name contains all underscore and no dash (for eg. when the lang value is en-GB)
                    //The reason to do this is because the script fails with exception in case we have "-" in field name.
                    fieldToAdd = normalizeFieldName(field, entity.getLanguage());
                } else {
                    fieldToAdd = field;
                }
                fieldValueToAdd = (String) entity.get(field);
                builder = builder.field(fieldToAdd, Arrays.asList(fieldValueToAdd));
                break;
            } else {
                //We are making sure that the field is at-least added to the index.
                //This will help us tremendously when we are appending the data later in an already indexed document.
                builder = builder.field(field, Collections.emptyList());
            }
        }
        builder = builder.endObject();
        IndexRequest indexRequest = new IndexRequest(indexName, docDef.getEntityField(), entity.getId()).source(builder);
        String addUpdateScript = ADD_UPDATE_SCRIPT.replaceAll("<fieldName>", fieldToAdd);
        Map<String, Object> params = new HashMap<>();
        params.put("fieldValue", fieldValueToAdd);
        UpdateRequest upReq = new UpdateRequest(indexName, docDef.getEntityField(), entity.getId()).script(new Script(Script.DEFAULT_SCRIPT_TYPE, Script.DEFAULT_SCRIPT_LANG, addUpdateScript, params)).upsert(indexRequest);
        UpdateResponse response = client.update(upReq).get();
        LOGGER.debug("Received the following Update response : " + response + " for the following entity: " + entity);
    } catch (Exception e) {
        throw new TextIndexException("Unable to Index the Entity in ElasticSearch.", e);
    }
}