Examples with IndexRequest org.elasticsearch.action.index.IndexRequest used on opensource projects

Example 86 with IndexRequest

use of org.elasticsearch.action.index.IndexRequest in project titan by thinkaurelius.

the class ElasticSearchIndex method mutate.

@Override
public void mutate(Map<String, Map<String, IndexMutation>> mutations, KeyInformation.IndexRetriever informations, TransactionHandle tx) throws StorageException {
    BulkRequestBuilder brb = client.prepareBulk();
    int bulkrequests = 0;
    try {
        for (Map.Entry<String, Map<String, IndexMutation>> stores : mutations.entrySet()) {
            String storename = stores.getKey();
            for (Map.Entry<String, IndexMutation> entry : stores.getValue().entrySet()) {
                String docid = entry.getKey();
                IndexMutation mutation = entry.getValue();
                Preconditions.checkArgument(!(mutation.isNew() && mutation.isDeleted()));
                Preconditions.checkArgument(!mutation.isNew() || !mutation.hasDeletions());
                Preconditions.checkArgument(!mutation.isDeleted() || !mutation.hasAdditions());
                // Deletions first
                if (mutation.hasDeletions()) {
                    if (mutation.isDeleted()) {
                        log.trace("Deleting entire document {}", docid);
                        brb.add(new DeleteRequest(indexName, storename, docid));
                        bulkrequests++;
                    } else {
                        Set<String> deletions = Sets.newHashSet(mutation.getDeletions());
                        if (mutation.hasAdditions()) {
                            for (IndexEntry ie : mutation.getAdditions()) {
                                deletions.remove(ie.key);
                            }
                        }
                        if (!deletions.isEmpty()) {
                            // TODO make part of batch mutation if/when possible
                            StringBuilder script = new StringBuilder();
                            for (String key : deletions) {
                                script.append("ctx._source.remove(\"" + key + "\"); ");
                            }
                            log.trace("Deleting individual fields [{}] for document {}", deletions, docid);
                            client.prepareUpdate(indexName, storename, docid).setScript(script.toString()).execute().actionGet();
                        }
                    }
                }
                if (mutation.hasAdditions()) {
                    if (mutation.isNew()) {
                        // Index
                        log.trace("Adding entire document {}", docid);
                        brb.add(new IndexRequest(indexName, storename, docid).source(getContent(mutation.getAdditions())));
                        bulkrequests++;
                    } else {
                        // Update: TODO make part of batch mutation if/when possible
                        boolean needUpsert = !mutation.hasDeletions();
                        XContentBuilder builder = getContent(mutation.getAdditions());
                        UpdateRequestBuilder update = client.prepareUpdate(indexName, storename, docid).setDoc(builder);
                        if (needUpsert)
                            update.setUpsert(builder);
                        log.trace("Updating document {} with upsert {}", docid, needUpsert);
                        update.execute().actionGet();
                    }
                }
            }
        }
        if (bulkrequests > 0)
            brb.execute().actionGet();
    } catch (Exception e) {
        throw convert(e);
    }
}

Example 87 with IndexRequest

use of org.elasticsearch.action.index.IndexRequest in project sonarqube by SonarSource.

the class EsRequestDetailsTest method should_format_IndexRequest.

@Test
public void should_format_IndexRequest() {
    IndexRequest indexRequest = new IndexRequest().index("index-1").type("type-1").id("id-1");
    assertThat(EsRequestDetails.computeDetailsAsString(indexRequest)).isEqualTo("ES index request for key 'id-1' on index 'index-1' on type 'type-1'");
}

Example 88 with IndexRequest

use of org.elasticsearch.action.index.IndexRequest in project snow-owl by b2ihealthcare.

the class RestHighLevelClientExt method bulk.

static Request bulk(BulkRequest bulkRequest) throws IOException {
    // Bulk API only supports newline delimited JSON or Smile. Before executing
    // the bulk, we need to check that all requests have the same content-type
    // and this content-type is supported by the Bulk API.
    XContentType bulkContentType = null;
    String index = null;
    for (int i = 0; i < bulkRequest.numberOfActions(); i++) {
        DocWriteRequest<?> writeRequest = bulkRequest.requests().get(i);
        index = enforceSameIndex(writeRequest.index(), index);
        // Remove index property, as it will be encoded in the request path
        DocWriteRequest.OpType opType = writeRequest.opType();
        switch(opType) {
            // $FALL-THROUGH$
            case INDEX:
            case CREATE:
                bulkContentType = enforceSameContentType((IndexRequest) writeRequest, bulkContentType);
                ((IndexRequest) writeRequest).index(null);
                break;
            case DELETE:
                ((DeleteRequest) writeRequest).index(null);
                break;
            case UPDATE:
                UpdateRequest updateRequest = (UpdateRequest) writeRequest;
                if (updateRequest.doc() != null) {
                    bulkContentType = enforceSameContentType(updateRequest.doc(), bulkContentType);
                }
                if (updateRequest.upsertRequest() != null) {
                    bulkContentType = enforceSameContentType(updateRequest.upsertRequest(), bulkContentType);
                }
                updateRequest.index(null);
                break;
        }
    }
    if (bulkContentType == null) {
        bulkContentType = XContentType.JSON;
    }
    String endpoint = endpoint(index, "_bulk");
    Request request = new Request(HttpPost.METHOD_NAME, endpoint);
    if (bulkRequest.timeout() != null) {
        request.addParameter("timeout", bulkRequest.timeout().getStringRep());
    }
    if (bulkRequest.getRefreshPolicy() != WriteRequest.RefreshPolicy.NONE) {
        request.addParameter("refresh", bulkRequest.getRefreshPolicy().getValue());
    }
    final byte separator = bulkContentType.xContent().streamSeparator();
    final ContentType requestContentType = RequestConverters.createContentType(bulkContentType);
    ByteArrayOutputStream content = new ByteArrayOutputStream();
    for (DocWriteRequest<?> writeRequest : bulkRequest.requests()) {
        DocWriteRequest.OpType opType = writeRequest.opType();
        try (XContentBuilder metadata = XContentBuilder.builder(bulkContentType.xContent())) {
            metadata.startObject();
            {
                metadata.startObject(opType.getLowercase());
                if (Strings.hasLength(writeRequest.index())) {
                    metadata.field("_index", writeRequest.index());
                }
                if (Strings.hasLength(writeRequest.id())) {
                    metadata.field("_id", writeRequest.id());
                }
                if (Strings.hasLength(writeRequest.routing())) {
                    metadata.field("routing", writeRequest.routing());
                }
                if (writeRequest.version() != Versions.MATCH_ANY) {
                    metadata.field("version", writeRequest.version());
                }
                VersionType versionType = writeRequest.versionType();
                if (versionType != VersionType.INTERNAL) {
                    if (versionType == VersionType.EXTERNAL) {
                        metadata.field("version_type", "external");
                    } else if (versionType == VersionType.EXTERNAL_GTE) {
                        metadata.field("version_type", "external_gte");
                    }
                }
                if (opType == DocWriteRequest.OpType.INDEX || opType == DocWriteRequest.OpType.CREATE) {
                    IndexRequest indexRequest = (IndexRequest) writeRequest;
                    if (Strings.hasLength(indexRequest.getPipeline())) {
                        metadata.field("pipeline", indexRequest.getPipeline());
                    }
                } else if (opType == DocWriteRequest.OpType.UPDATE) {
                    UpdateRequest updateRequest = (UpdateRequest) writeRequest;
                    if (updateRequest.retryOnConflict() > 0) {
                        metadata.field("retry_on_conflict", updateRequest.retryOnConflict());
                    }
                    if (updateRequest.fetchSource() != null) {
                        metadata.field("_source", updateRequest.fetchSource());
                    }
                }
                metadata.endObject();
            }
            metadata.endObject();
            BytesRef metadataSource = BytesReference.bytes(metadata).toBytesRef();
            content.write(metadataSource.bytes, metadataSource.offset, metadataSource.length);
            content.write(separator);
        }
        BytesRef source = null;
        if (opType == DocWriteRequest.OpType.INDEX || opType == DocWriteRequest.OpType.CREATE) {
            IndexRequest indexRequest = (IndexRequest) writeRequest;
            BytesReference indexSource = indexRequest.source();
            XContentType indexXContentType = indexRequest.getContentType();
            try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, indexSource, indexXContentType)) {
                try (XContentBuilder builder = XContentBuilder.builder(bulkContentType.xContent())) {
                    builder.copyCurrentStructure(parser);
                    source = BytesReference.bytes(builder).toBytesRef();
                }
            }
        } else if (opType == DocWriteRequest.OpType.UPDATE) {
            source = XContentHelper.toXContent((UpdateRequest) writeRequest, bulkContentType, false).toBytesRef();
        }
        if (source != null) {
            content.write(source.bytes, source.offset, source.length);
            content.write(separator);
        }
    }
    HttpEntity entity = new ByteArrayEntity(content.toByteArray(), 0, content.size(), requestContentType);
    request.setEntity(entity);
    return request;
}

Example 89 with IndexRequest

use of org.elasticsearch.action.index.IndexRequest in project elasticsearch by elastic.

the class TransportShardBulkAction method shardOperationOnReplica.

@Override
public WriteReplicaResult<BulkShardRequest> shardOperationOnReplica(BulkShardRequest request, IndexShard replica) throws Exception {
    Translog.Location location = null;
    for (int i = 0; i < request.items().length; i++) {
        BulkItemRequest item = request.items()[i];
        if (shouldExecuteReplicaItem(item, i)) {
            DocWriteRequest docWriteRequest = item.request();
            DocWriteResponse primaryResponse = item.getPrimaryResponse().getResponse();
            final Engine.Result operationResult;
            try {
                switch(docWriteRequest.opType()) {
                    case CREATE:
                    case INDEX:
                        operationResult = executeIndexRequestOnReplica(primaryResponse, (IndexRequest) docWriteRequest, replica);
                        break;
                    case DELETE:
                        operationResult = executeDeleteRequestOnReplica(primaryResponse, (DeleteRequest) docWriteRequest, replica);
                        break;
                    default:
                        throw new IllegalStateException("Unexpected request operation type on replica: " + docWriteRequest.opType().getLowercase());
                }
                if (operationResult.hasFailure()) {
                    // check if any transient write operation failures should be bubbled up
                    Exception failure = operationResult.getFailure();
                    assert failure instanceof VersionConflictEngineException || failure instanceof MapperParsingException : "expected any one of [version conflict, mapper parsing, engine closed, index shard closed]" + " failures. got " + failure;
                    if (!TransportActions.isShardNotAvailableException(failure)) {
                        throw failure;
                    }
                } else {
                    location = locationToSync(location, operationResult.getTranslogLocation());
                }
            } catch (Exception e) {
                // so we will fail the shard
                if (!TransportActions.isShardNotAvailableException(e)) {
                    throw e;
                }
            }
        }
    }
    return new WriteReplicaResult<>(request, location, null, replica, logger);
}

Example 90 with IndexRequest

use of org.elasticsearch.action.index.IndexRequest in project elasticsearch by elastic.

the class TransportShardBulkAction method executeUpdateRequest.

/**
     * Executes update request, delegating to a index or delete operation after translation,
     * handles retries on version conflict and constructs update response
     * NOTE: reassigns bulk item request at <code>requestIndex</code> for replicas to
     * execute translated update request (NOOP update is an exception). NOOP updates are
     * indicated by returning a <code>null</code> operation in {@link BulkItemResultHolder}
     * */
private static BulkItemResultHolder executeUpdateRequest(UpdateRequest updateRequest, IndexShard primary, IndexMetaData metaData, BulkShardRequest request, int requestIndex, UpdateHelper updateHelper, LongSupplier nowInMillis, final MappingUpdatePerformer mappingUpdater) throws Exception {
    Engine.Result updateOperationResult = null;
    UpdateResponse updateResponse = null;
    BulkItemRequest replicaRequest = request.items()[requestIndex];
    int maxAttempts = updateRequest.retryOnConflict();
    for (int attemptCount = 0; attemptCount <= maxAttempts; attemptCount++) {
        final UpdateHelper.Result translate;
        // translate update request
        try {
            translate = updateHelper.prepare(updateRequest, primary, nowInMillis);
        } catch (Exception failure) {
            // we may fail translating a update to index or delete operation
            // we use index result to communicate failure while translating update request
            updateOperationResult = new Engine.IndexResult(failure, updateRequest.version(), SequenceNumbersService.UNASSIGNED_SEQ_NO);
            // out of retry loop
            break;
        }
        // execute translated update request
        switch(translate.getResponseResult()) {
            case CREATED:
            case UPDATED:
                IndexRequest indexRequest = translate.action();
                MappingMetaData mappingMd = metaData.mappingOrDefault(indexRequest.type());
                indexRequest.process(mappingMd, request.index());
                updateOperationResult = executeIndexRequestOnPrimary(indexRequest, primary, mappingUpdater);
                break;
            case DELETED:
                DeleteRequest deleteRequest = translate.action();
                updateOperationResult = executeDeleteRequestOnPrimary(deleteRequest, primary);
                break;
            case NOOP:
                primary.noopUpdate(updateRequest.type());
                break;
            default:
                throw new IllegalStateException("Illegal update operation " + translate.getResponseResult());
        }
        if (updateOperationResult == null) {
            // this is a noop operation
            updateResponse = translate.action();
            // out of retry loop
            break;
        } else if (updateOperationResult.hasFailure() == false) {
            // set translated update (index/delete) request for replica execution in bulk items
            switch(updateOperationResult.getOperationType()) {
                case INDEX:
                    IndexRequest updateIndexRequest = translate.action();
                    final IndexResponse indexResponse = new IndexResponse(primary.shardId(), updateIndexRequest.type(), updateIndexRequest.id(), updateOperationResult.getSeqNo(), updateOperationResult.getVersion(), ((Engine.IndexResult) updateOperationResult).isCreated());
                    BytesReference indexSourceAsBytes = updateIndexRequest.source();
                    updateResponse = new UpdateResponse(indexResponse.getShardInfo(), indexResponse.getShardId(), indexResponse.getType(), indexResponse.getId(), indexResponse.getSeqNo(), indexResponse.getVersion(), indexResponse.getResult());
                    if ((updateRequest.fetchSource() != null && updateRequest.fetchSource().fetchSource()) || (updateRequest.fields() != null && updateRequest.fields().length > 0)) {
                        Tuple<XContentType, Map<String, Object>> sourceAndContent = XContentHelper.convertToMap(indexSourceAsBytes, true, updateIndexRequest.getContentType());
                        updateResponse.setGetResult(updateHelper.extractGetResult(updateRequest, request.index(), indexResponse.getVersion(), sourceAndContent.v2(), sourceAndContent.v1(), indexSourceAsBytes));
                    }
                    // set translated request as replica request
                    replicaRequest = new BulkItemRequest(request.items()[requestIndex].id(), updateIndexRequest);
                    break;
                case DELETE:
                    DeleteRequest updateDeleteRequest = translate.action();
                    DeleteResponse deleteResponse = new DeleteResponse(primary.shardId(), updateDeleteRequest.type(), updateDeleteRequest.id(), updateOperationResult.getSeqNo(), updateOperationResult.getVersion(), ((Engine.DeleteResult) updateOperationResult).isFound());
                    updateResponse = new UpdateResponse(deleteResponse.getShardInfo(), deleteResponse.getShardId(), deleteResponse.getType(), deleteResponse.getId(), deleteResponse.getSeqNo(), deleteResponse.getVersion(), deleteResponse.getResult());
                    updateResponse.setGetResult(updateHelper.extractGetResult(updateRequest, request.index(), deleteResponse.getVersion(), translate.updatedSourceAsMap(), translate.updateSourceContentType(), null));
                    // set translated request as replica request
                    replicaRequest = new BulkItemRequest(request.items()[requestIndex].id(), updateDeleteRequest);
                    break;
            }
            assert updateOperationResult.getSeqNo() != SequenceNumbersService.UNASSIGNED_SEQ_NO;
            // out of retry loop
            break;
        } else if (updateOperationResult.getFailure() instanceof VersionConflictEngineException == false) {
            // out of retry loop
            break;
        }
    }
    return new BulkItemResultHolder(updateResponse, updateOperationResult, replicaRequest);
}