Examples with PutPipelineRequest org.opensearch.action.ingest.PutPipelineRequest used on opensource projects

Example 11 with PutPipelineRequest

use of org.opensearch.action.ingest.PutPipelineRequest in project OpenSearch by opensearch-project.

the class IngestServiceTests method testPut.

public void testPut() {
    IngestService ingestService = createWithProcessors();
    String id = "_id";
    Pipeline pipeline = ingestService.getPipeline(id);
    assertThat(pipeline, nullValue());
    ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).build();
    // add a new pipeline:
    PutPipelineRequest putRequest = new PutPipelineRequest(id, new BytesArray("{\"processors\": []}"), XContentType.JSON);
    ClusterState previousClusterState = clusterState;
    clusterState = IngestService.innerPut(putRequest, clusterState);
    ingestService.applyClusterState(new ClusterChangedEvent("", clusterState, previousClusterState));
    pipeline = ingestService.getPipeline(id);
    assertThat(pipeline, notNullValue());
    assertThat(pipeline.getId(), equalTo(id));
    assertThat(pipeline.getDescription(), nullValue());
    assertThat(pipeline.getProcessors().size(), equalTo(0));
    // overwrite existing pipeline:
    putRequest = new PutPipelineRequest(id, new BytesArray("{\"processors\": [], \"description\": \"_description\"}"), XContentType.JSON);
    previousClusterState = clusterState;
    clusterState = IngestService.innerPut(putRequest, clusterState);
    ingestService.applyClusterState(new ClusterChangedEvent("", clusterState, previousClusterState));
    pipeline = ingestService.getPipeline(id);
    assertThat(pipeline, notNullValue());
    assertThat(pipeline.getId(), equalTo(id));
    assertThat(pipeline.getDescription(), equalTo("_description"));
    assertThat(pipeline.getProcessors().size(), equalTo(0));
}

Example 12 with PutPipelineRequest

use of org.opensearch.action.ingest.PutPipelineRequest in project OpenSearch by opensearch-project.

the class IngestServiceTests method testCrud.

public void testCrud() throws Exception {
    IngestService ingestService = createWithProcessors();
    String id = "_id";
    Pipeline pipeline = ingestService.getPipeline(id);
    assertThat(pipeline, nullValue());
    // Start empty
    ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).build();
    PutPipelineRequest putRequest = new PutPipelineRequest(id, new BytesArray("{\"processors\": [{\"set\" : {\"field\": \"_field\", \"value\": \"_value\"}}]}"), XContentType.JSON);
    ClusterState previousClusterState = clusterState;
    clusterState = IngestService.innerPut(putRequest, clusterState);
    ingestService.applyClusterState(new ClusterChangedEvent("", clusterState, previousClusterState));
    pipeline = ingestService.getPipeline(id);
    assertThat(pipeline, notNullValue());
    assertThat(pipeline.getId(), equalTo(id));
    assertThat(pipeline.getDescription(), nullValue());
    assertThat(pipeline.getProcessors().size(), equalTo(1));
    assertThat(pipeline.getProcessors().get(0).getType(), equalTo("set"));
    DeletePipelineRequest deleteRequest = new DeletePipelineRequest(id);
    previousClusterState = clusterState;
    clusterState = IngestService.innerDelete(deleteRequest, clusterState);
    ingestService.applyClusterState(new ClusterChangedEvent("", clusterState, previousClusterState));
    pipeline = ingestService.getPipeline(id);
    assertThat(pipeline, nullValue());
}

Example 13 with PutPipelineRequest

use of org.opensearch.action.ingest.PutPipelineRequest in project OpenSearch by opensearch-project.

the class IngestServiceTests method testExecuteIndexPipelineExistsButFailedParsing.

public void testExecuteIndexPipelineExistsButFailedParsing() {
    IngestService ingestService = createWithProcessors(Collections.singletonMap("mock", (factories, tag, description, config) -> new AbstractProcessor("mock", "description") {

        @Override
        public IngestDocument execute(IngestDocument ingestDocument) {
            throw new IllegalStateException("error");
        }

        @Override
        public String getType() {
            return null;
        }
    }));
    // Start empty
    ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).build();
    String id = "_id";
    PutPipelineRequest putRequest = new PutPipelineRequest(id, new BytesArray("{\"processors\": [{\"mock\" : {}}]}"), XContentType.JSON);
    ClusterState previousClusterState = clusterState;
    clusterState = IngestService.innerPut(putRequest, clusterState);
    ingestService.applyClusterState(new ClusterChangedEvent("", clusterState, previousClusterState));
    final SetOnce<Boolean> failure = new SetOnce<>();
    BulkRequest bulkRequest = new BulkRequest();
    final IndexRequest indexRequest1 = new IndexRequest("_index").id("_id1").source(emptyMap()).setPipeline("_none").setFinalPipeline("_none");
    bulkRequest.add(indexRequest1);
    IndexRequest indexRequest2 = new IndexRequest("_index").id("_id2").source(emptyMap()).setPipeline(id).setFinalPipeline("_none");
    bulkRequest.add(indexRequest2);
    final BiConsumer<Integer, Exception> failureHandler = (slot, e) -> {
        assertThat(e.getCause(), instanceOf(IllegalStateException.class));
        assertThat(e.getCause().getMessage(), equalTo("error"));
        failure.set(true);
        assertThat(slot, equalTo(1));
    };
    @SuppressWarnings("unchecked") final BiConsumer<Thread, Exception> completionHandler = mock(BiConsumer.class);
    ingestService.executeBulkRequest(bulkRequest.numberOfActions(), bulkRequest.requests(), failureHandler, completionHandler, indexReq -> {
    }, Names.WRITE);
    assertTrue(failure.get());
    verify(completionHandler, times(1)).accept(Thread.currentThread(), null);
}

Example 14 with PutPipelineRequest

use of org.opensearch.action.ingest.PutPipelineRequest in project OpenSearch by opensearch-project.

the class IngestServiceTests method testGetProcessorsInPipelineComplexConditional.

public void testGetProcessorsInPipelineComplexConditional() throws Exception {
    LongSupplier relativeTimeProvider = mock(LongSupplier.class);
    String scriptName = "conditionalScript";
    ScriptService scriptService = new ScriptService(Settings.builder().build(), Collections.singletonMap(Script.DEFAULT_SCRIPT_LANG, new MockScriptEngine(Script.DEFAULT_SCRIPT_LANG, Collections.singletonMap(scriptName, ctx -> {
        ctx.get("_type");
        return true;
    }), Collections.emptyMap())), new HashMap<>(ScriptModule.CORE_CONTEXTS));
    Map<String, Processor.Factory> processors = new HashMap<>();
    processors.put("complexSet", (factories, tag, description, config) -> {
        String field = (String) config.remove("field");
        String value = (String) config.remove("value");
        return new ConditionalProcessor(randomAlphaOfLength(10), null, new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, scriptName, Collections.emptyMap()), scriptService, new ConditionalProcessor(randomAlphaOfLength(10) + "-nested", null, new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, scriptName, Collections.emptyMap()), scriptService, new FakeProcessor("complexSet", tag, description, (ingestDocument) -> ingestDocument.setFieldValue(field, value))));
    });
    IngestService ingestService = createWithProcessors(processors);
    String id = "_id";
    Pipeline pipeline = ingestService.getPipeline(id);
    assertThat(pipeline, nullValue());
    // Start empty
    ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).build();
    PutPipelineRequest putRequest = new PutPipelineRequest(id, new BytesArray("{\"processors\": [{\"complexSet\" : {\"field\": \"_field\", \"value\": \"_value\"}}]}"), XContentType.JSON);
    ClusterState previousClusterState = clusterState;
    clusterState = IngestService.innerPut(putRequest, clusterState);
    ingestService.applyClusterState(new ClusterChangedEvent("", clusterState, previousClusterState));
    pipeline = ingestService.getPipeline(id);
    assertThat(pipeline, notNullValue());
    assertThat(ingestService.getProcessorsInPipeline(id, Processor.class).size(), equalTo(3));
    assertThat(ingestService.getProcessorsInPipeline(id, WrappingProcessor.class).size(), equalTo(2));
    assertThat(ingestService.getProcessorsInPipeline(id, FakeProcessor.class).size(), equalTo(1));
    assertThat(ingestService.getProcessorsInPipeline(id, ConditionalProcessor.class).size(), equalTo(2));
    assertThat(ingestService.getProcessorsInPipeline(id, WrappingProcessorImpl.class).size(), equalTo(0));
}

Example 15 with PutPipelineRequest

use of org.opensearch.action.ingest.PutPipelineRequest in project OpenSearch by opensearch-project.

the class IngestServiceTests method testBulkRequestExecutionWithFailures.

public void testBulkRequestExecutionWithFailures() throws Exception {
    BulkRequest bulkRequest = new BulkRequest();
    String pipelineId = "_id";
    int numRequest = scaledRandomIntBetween(8, 64);
    int numIndexRequests = 0;
    for (int i = 0; i < numRequest; i++) {
        DocWriteRequest request;
        if (randomBoolean()) {
            if (randomBoolean()) {
                request = new DeleteRequest("_index", "_id");
            } else {
                request = new UpdateRequest("_index", "_id");
            }
        } else {
            IndexRequest indexRequest = new IndexRequest("_index").id("_id").setPipeline(pipelineId).setFinalPipeline("_none");
            indexRequest.source(Requests.INDEX_CONTENT_TYPE, "field1", "value1");
            request = indexRequest;
            numIndexRequests++;
        }
        bulkRequest.add(request);
    }
    CompoundProcessor processor = mock(CompoundProcessor.class);
    when(processor.getProcessors()).thenReturn(Collections.singletonList(mock(Processor.class)));
    Exception error = new RuntimeException();
    doAnswer(args -> {
        @SuppressWarnings("unchecked") BiConsumer<IngestDocument, Exception> handler = (BiConsumer) args.getArguments()[1];
        handler.accept(null, error);
        return null;
    }).when(processor).execute(any(), any());
    IngestService ingestService = createWithProcessors(Collections.singletonMap("mock", (factories, tag, description, config) -> processor));
    PutPipelineRequest putRequest = new PutPipelineRequest("_id", new BytesArray("{\"processors\": [{\"mock\" : {}}]}"), XContentType.JSON);
    // Start empty
    ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).build();
    ClusterState previousClusterState = clusterState;
    clusterState = IngestService.innerPut(putRequest, clusterState);
    ingestService.applyClusterState(new ClusterChangedEvent("", clusterState, previousClusterState));
    @SuppressWarnings("unchecked") BiConsumer<Integer, Exception> requestItemErrorHandler = mock(BiConsumer.class);
    @SuppressWarnings("unchecked") final BiConsumer<Thread, Exception> completionHandler = mock(BiConsumer.class);
    ingestService.executeBulkRequest(numRequest, bulkRequest.requests(), requestItemErrorHandler, completionHandler, indexReq -> {
    }, Names.WRITE);
    verify(requestItemErrorHandler, times(numIndexRequests)).accept(anyInt(), argThat(o -> o.getCause().equals(error)));
    verify(completionHandler, times(1)).accept(Thread.currentThread(), null);
}