Examples with UpdateHelper org.opensearch.action.update.UpdateHelper used on opensource projects

Example 1 with UpdateHelper

use of org.opensearch.action.update.UpdateHelper in project OpenSearch by opensearch-project.

the class TransportShardBulkAction method executeBulkItemRequest.

/**
 * Executes bulk item requests and handles request execution exceptions.
 * @return {@code true} if request completed on this thread and the listener was invoked, {@code false} if the request triggered
 *                      a mapping update that will finish and invoke the listener on a different thread
 */
static boolean executeBulkItemRequest(BulkPrimaryExecutionContext context, UpdateHelper updateHelper, LongSupplier nowInMillisSupplier, MappingUpdatePerformer mappingUpdater, Consumer<ActionListener<Void>> waitForMappingUpdate, ActionListener<Void> itemDoneListener) throws Exception {
    final DocWriteRequest.OpType opType = context.getCurrent().opType();
    final UpdateHelper.Result updateResult;
    if (opType == DocWriteRequest.OpType.UPDATE) {
        final UpdateRequest updateRequest = (UpdateRequest) context.getCurrent();
        try {
            updateResult = updateHelper.prepare(updateRequest, context.getPrimary(), nowInMillisSupplier);
        } catch (Exception failure) {
            // we may fail translating a update to index or delete operation
            // we use index result to communicate failure while translating update request
            final Engine.Result result = new Engine.IndexResult(failure, updateRequest.version());
            context.setRequestToExecute(updateRequest);
            context.markOperationAsExecuted(result);
            context.markAsCompleted(context.getExecutionResult());
            return true;
        }
        // execute translated update request
        switch(updateResult.getResponseResult()) {
            case CREATED:
            case UPDATED:
                IndexRequest indexRequest = updateResult.action();
                IndexMetadata metadata = context.getPrimary().indexSettings().getIndexMetadata();
                MappingMetadata mappingMd = metadata.mapping();
                indexRequest.process(metadata.getCreationVersion(), mappingMd, updateRequest.concreteIndex());
                context.setRequestToExecute(indexRequest);
                break;
            case DELETED:
                context.setRequestToExecute(updateResult.action());
                break;
            case NOOP:
                context.markOperationAsNoOp(updateResult.action());
                context.markAsCompleted(context.getExecutionResult());
                return true;
            default:
                throw new IllegalStateException("Illegal update operation " + updateResult.getResponseResult());
        }
    } else {
        context.setRequestToExecute(context.getCurrent());
        updateResult = null;
    }
    // also checks that we're in TRANSLATED state
    assert context.getRequestToExecute() != null;
    final IndexShard primary = context.getPrimary();
    final long version = context.getRequestToExecute().version();
    final boolean isDelete = context.getRequestToExecute().opType() == DocWriteRequest.OpType.DELETE;
    final Engine.Result result;
    if (isDelete) {
        final DeleteRequest request = context.getRequestToExecute();
        result = primary.applyDeleteOperationOnPrimary(version, MapperService.SINGLE_MAPPING_NAME, request.id(), request.versionType(), request.ifSeqNo(), request.ifPrimaryTerm());
    } else {
        final IndexRequest request = context.getRequestToExecute();
        result = primary.applyIndexOperationOnPrimary(version, request.versionType(), new SourceToParse(request.index(), MapperService.SINGLE_MAPPING_NAME, request.id(), request.source(), request.getContentType(), request.routing()), request.ifSeqNo(), request.ifPrimaryTerm(), request.getAutoGeneratedTimestamp(), request.isRetry());
    }
    if (result.getResultType() == Engine.Result.Type.MAPPING_UPDATE_REQUIRED) {
        try {
            primary.mapperService().merge(MapperService.SINGLE_MAPPING_NAME, new CompressedXContent(result.getRequiredMappingUpdate(), ToXContent.EMPTY_PARAMS), MapperService.MergeReason.MAPPING_UPDATE_PREFLIGHT);
        } catch (Exception e) {
            logger.info(() -> new ParameterizedMessage("{} mapping update rejected by primary", primary.shardId()), e);
            onComplete(exceptionToResult(e, primary, isDelete, version), context, updateResult);
            return true;
        }
        mappingUpdater.updateMappings(result.getRequiredMappingUpdate(), primary.shardId(), new ActionListener<Void>() {

            @Override
            public void onResponse(Void v) {
                context.markAsRequiringMappingUpdate();
                waitForMappingUpdate.accept(ActionListener.runAfter(new ActionListener<Void>() {

                    @Override
                    public void onResponse(Void v) {
                        assert context.requiresWaitingForMappingUpdate();
                        context.resetForExecutionForRetry();
                    }

                    @Override
                    public void onFailure(Exception e) {
                        context.failOnMappingUpdate(e);
                    }
                }, () -> itemDoneListener.onResponse(null)));
            }

            @Override
            public void onFailure(Exception e) {
                onComplete(exceptionToResult(e, primary, isDelete, version), context, updateResult);
                // Requesting mapping update failed, so we don't have to wait for a cluster state update
                assert context.isInitial();
                itemDoneListener.onResponse(null);
            }
        });
        return false;
    } else {
        onComplete(result, context, updateResult);
    }
    return true;
}

Example 2 with UpdateHelper

use of org.opensearch.action.update.UpdateHelper in project OpenSearch by opensearch-project.

the class TransportShardBulkAction method performOnPrimary.

public static void performOnPrimary(BulkShardRequest request, IndexShard primary, UpdateHelper updateHelper, LongSupplier nowInMillisSupplier, MappingUpdatePerformer mappingUpdater, Consumer<ActionListener<Void>> waitForMappingUpdate, ActionListener<PrimaryResult<BulkShardRequest, BulkShardResponse>> listener, ThreadPool threadPool, String executorName) {
    new ActionRunnable<PrimaryResult<BulkShardRequest, BulkShardResponse>>(listener) {

        private final Executor executor = threadPool.executor(executorName);

        private final BulkPrimaryExecutionContext context = new BulkPrimaryExecutionContext(request, primary);

        @Override
        protected void doRun() throws Exception {
            while (context.hasMoreOperationsToExecute()) {
                if (executeBulkItemRequest(context, updateHelper, nowInMillisSupplier, mappingUpdater, waitForMappingUpdate, ActionListener.wrap(v -> executor.execute(this), this::onRejection)) == false) {
                    // so we just break out here.
                    return;
                }
                // either completed and moved to next or reset
                assert context.isInitial();
            }
            // We're done, there's no more operations to execute so we resolve the wrapped listener
            finishRequest();
        }

        @Override
        public void onRejection(Exception e) {
            // We must finish the outstanding request. Finishing the outstanding request can include
            // refreshing and fsyncing. Therefore, we must force execution on the WRITE thread.
            executor.execute(new ActionRunnable<PrimaryResult<BulkShardRequest, BulkShardResponse>>(listener) {

                @Override
                protected void doRun() {
                    // Fail all operations after a bulk rejection hit an action that waited for a mapping update and finish the request
                    while (context.hasMoreOperationsToExecute()) {
                        context.setRequestToExecute(context.getCurrent());
                        final DocWriteRequest<?> docWriteRequest = context.getRequestToExecute();
                        onComplete(exceptionToResult(e, primary, docWriteRequest.opType() == DocWriteRequest.OpType.DELETE, docWriteRequest.version()), context, null);
                    }
                    finishRequest();
                }

                @Override
                public boolean isForceExecution() {
                    return true;
                }
            });
        }

        private void finishRequest() {
            ActionListener.completeWith(listener, () -> new WritePrimaryResult<>(context.getBulkShardRequest(), context.buildShardResponse(), context.getLocationToSync(), null, context.getPrimary(), logger));
        }
    }.run();
}

Example 3 with UpdateHelper

use of org.opensearch.action.update.UpdateHelper in project OpenSearch by opensearch-project.

the class TransportShardBulkActionTests method testUpdateRequestWithSuccess.

public void testUpdateRequestWithSuccess() throws Exception {
    IndexSettings indexSettings = new IndexSettings(indexMetadata(), Settings.EMPTY);
    DocWriteRequest<UpdateRequest> writeRequest = new UpdateRequest("index", "id").doc(Requests.INDEX_CONTENT_TYPE, "field", "value");
    BulkItemRequest primaryRequest = new BulkItemRequest(0, writeRequest);
    IndexRequest updateResponse = new IndexRequest("index").id("id").source(Requests.INDEX_CONTENT_TYPE, "field", "value");
    boolean created = randomBoolean();
    Translog.Location resultLocation = new Translog.Location(42, 42, 42);
    Engine.IndexResult indexResult = new FakeIndexResult(1, 1, 13, created, resultLocation);
    IndexShard shard = mock(IndexShard.class);
    when(shard.applyIndexOperationOnPrimary(anyLong(), any(), any(), anyLong(), anyLong(), anyLong(), anyBoolean())).thenReturn(indexResult);
    when(shard.indexSettings()).thenReturn(indexSettings);
    when(shard.shardId()).thenReturn(shardId);
    UpdateHelper updateHelper = mock(UpdateHelper.class);
    when(updateHelper.prepare(any(), eq(shard), any())).thenReturn(new UpdateHelper.Result(updateResponse, created ? DocWriteResponse.Result.CREATED : DocWriteResponse.Result.UPDATED, Collections.singletonMap("field", "value"), Requests.INDEX_CONTENT_TYPE));
    BulkItemRequest[] items = new BulkItemRequest[] { primaryRequest };
    BulkShardRequest bulkShardRequest = new BulkShardRequest(shardId, RefreshPolicy.NONE, items);
    randomlySetIgnoredPrimaryResponse(primaryRequest);
    BulkPrimaryExecutionContext context = new BulkPrimaryExecutionContext(bulkShardRequest, shard);
    TransportShardBulkAction.executeBulkItemRequest(context, updateHelper, threadPool::absoluteTimeInMillis, new NoopMappingUpdatePerformer(), listener -> {
    }, ASSERTING_DONE_LISTENER);
    assertFalse(context.hasMoreOperationsToExecute());
    // Check that the translog is successfully advanced
    assertThat(context.getLocationToSync(), equalTo(resultLocation));
    assertThat(bulkShardRequest.items()[0].request(), equalTo(updateResponse));
    // Since this was not a conflict failure, the primary response
    // should be filled out with the failure information
    BulkItemResponse primaryResponse = bulkShardRequest.items()[0].getPrimaryResponse();
    assertThat(primaryResponse.getItemId(), equalTo(0));
    assertThat(primaryResponse.getId(), equalTo("id"));
    assertThat(primaryResponse.getOpType(), equalTo(DocWriteRequest.OpType.UPDATE));
    DocWriteResponse response = primaryResponse.getResponse();
    assertThat(response.status(), equalTo(created ? RestStatus.CREATED : RestStatus.OK));
    assertThat(response.getSeqNo(), equalTo(13L));
}

Example 4 with UpdateHelper

use of org.opensearch.action.update.UpdateHelper in project OpenSearch by opensearch-project.

the class TransportShardBulkActionTests method testRetries.

public void testRetries() throws Exception {
    IndexSettings indexSettings = new IndexSettings(indexMetadata(), Settings.EMPTY);
    UpdateRequest writeRequest = new UpdateRequest("index", "id").doc(Requests.INDEX_CONTENT_TYPE, "field", "value");
    // the beating will continue until success has come.
    writeRequest.retryOnConflict(Integer.MAX_VALUE);
    BulkItemRequest primaryRequest = new BulkItemRequest(0, writeRequest);
    IndexRequest updateResponse = new IndexRequest("index").id("id").source(Requests.INDEX_CONTENT_TYPE, "field", "value");
    Exception err = new VersionConflictEngineException(shardId, "id", "I'm conflicted <(;_;)>");
    Engine.IndexResult conflictedResult = new Engine.IndexResult(err, 0);
    Engine.IndexResult mappingUpdate = new Engine.IndexResult(new Mapping(null, mock(RootObjectMapper.class), new MetadataFieldMapper[0], Collections.emptyMap()));
    Translog.Location resultLocation = new Translog.Location(42, 42, 42);
    Engine.IndexResult success = new FakeIndexResult(1, 1, 13, true, resultLocation);
    IndexShard shard = mock(IndexShard.class);
    when(shard.applyIndexOperationOnPrimary(anyLong(), any(), any(), anyLong(), anyLong(), anyLong(), anyBoolean())).thenAnswer(ir -> {
        if (randomBoolean()) {
            return conflictedResult;
        }
        if (randomBoolean()) {
            return mappingUpdate;
        } else {
            return success;
        }
    });
    when(shard.indexSettings()).thenReturn(indexSettings);
    when(shard.shardId()).thenReturn(shardId);
    when(shard.mapperService()).thenReturn(mock(MapperService.class));
    UpdateHelper updateHelper = mock(UpdateHelper.class);
    when(updateHelper.prepare(any(), eq(shard), any())).thenReturn(new UpdateHelper.Result(updateResponse, randomBoolean() ? DocWriteResponse.Result.CREATED : DocWriteResponse.Result.UPDATED, Collections.singletonMap("field", "value"), Requests.INDEX_CONTENT_TYPE));
    BulkItemRequest[] items = new BulkItemRequest[] { primaryRequest };
    BulkShardRequest bulkShardRequest = new BulkShardRequest(shardId, RefreshPolicy.NONE, items);
    final CountDownLatch latch = new CountDownLatch(1);
    TransportShardBulkAction.performOnPrimary(bulkShardRequest, shard, updateHelper, threadPool::absoluteTimeInMillis, new NoopMappingUpdatePerformer(), listener -> listener.onResponse(null), new LatchedActionListener<>(ActionTestUtils.assertNoFailureListener(result -> {
        assertThat(((WritePrimaryResult<BulkShardRequest, BulkShardResponse>) result).location, equalTo(resultLocation));
        BulkItemResponse primaryResponse = result.replicaRequest().items()[0].getPrimaryResponse();
        assertThat(primaryResponse.getItemId(), equalTo(0));
        assertThat(primaryResponse.getId(), equalTo("id"));
        assertThat(primaryResponse.getOpType(), equalTo(DocWriteRequest.OpType.UPDATE));
        DocWriteResponse response = primaryResponse.getResponse();
        assertThat(response.status(), equalTo(RestStatus.CREATED));
        assertThat(response.getSeqNo(), equalTo(13L));
    }), latch), threadPool, Names.WRITE);
    latch.await();
}

Example 5 with UpdateHelper

use of org.opensearch.action.update.UpdateHelper in project OpenSearch by opensearch-project.

the class TransportShardBulkActionTests method testUpdateWithDelete.

public void testUpdateWithDelete() throws Exception {
    IndexSettings indexSettings = new IndexSettings(indexMetadata(), Settings.EMPTY);
    DocWriteRequest<UpdateRequest> writeRequest = new UpdateRequest("index", "id").doc(Requests.INDEX_CONTENT_TYPE, "field", "value");
    BulkItemRequest primaryRequest = new BulkItemRequest(0, writeRequest);
    DeleteRequest updateResponse = new DeleteRequest("index", "id");
    boolean found = randomBoolean();
    Translog.Location resultLocation = new Translog.Location(42, 42, 42);
    final long resultSeqNo = 13;
    Engine.DeleteResult deleteResult = new FakeDeleteResult(1, 1, resultSeqNo, found, resultLocation);
    IndexShard shard = mock(IndexShard.class);
    when(shard.applyDeleteOperationOnPrimary(anyLong(), any(), any(), any(), anyLong(), anyLong())).thenReturn(deleteResult);
    when(shard.indexSettings()).thenReturn(indexSettings);
    when(shard.shardId()).thenReturn(shardId);
    UpdateHelper updateHelper = mock(UpdateHelper.class);
    when(updateHelper.prepare(any(), eq(shard), any())).thenReturn(new UpdateHelper.Result(updateResponse, DocWriteResponse.Result.DELETED, Collections.singletonMap("field", "value"), Requests.INDEX_CONTENT_TYPE));
    BulkItemRequest[] items = new BulkItemRequest[] { primaryRequest };
    BulkShardRequest bulkShardRequest = new BulkShardRequest(shardId, RefreshPolicy.NONE, items);
    randomlySetIgnoredPrimaryResponse(primaryRequest);
    BulkPrimaryExecutionContext context = new BulkPrimaryExecutionContext(bulkShardRequest, shard);
    TransportShardBulkAction.executeBulkItemRequest(context, updateHelper, threadPool::absoluteTimeInMillis, new NoopMappingUpdatePerformer(), listener -> listener.onResponse(null), ASSERTING_DONE_LISTENER);
    assertFalse(context.hasMoreOperationsToExecute());
    // Check that the translog is successfully advanced
    assertThat(context.getLocationToSync(), equalTo(resultLocation));
    assertThat(bulkShardRequest.items()[0].request(), equalTo(updateResponse));
    BulkItemResponse primaryResponse = bulkShardRequest.items()[0].getPrimaryResponse();
    assertThat(primaryResponse.getItemId(), equalTo(0));
    assertThat(primaryResponse.getId(), equalTo("id"));
    assertThat(primaryResponse.getOpType(), equalTo(DocWriteRequest.OpType.UPDATE));
    DocWriteResponse response = primaryResponse.getResponse();
    assertThat(response.status(), equalTo(RestStatus.OK));
    assertThat(response.getSeqNo(), equalTo(resultSeqNo));
}