Examples with PutPipelineRequest org.opensearch.action.ingest.PutPipelineRequest used on opensource projects

Example 36 with PutPipelineRequest

use of org.opensearch.action.ingest.PutPipelineRequest in project OpenSearch by opensearch-project.

the class IngestServiceTests method testExecuteSuccessWithOnFailure.

public void testExecuteSuccessWithOnFailure() throws Exception {
    final Processor processor = mock(Processor.class);
    when(processor.getType()).thenReturn("mock_processor_type");
    when(processor.getTag()).thenReturn("mock_processor_tag");
    doAnswer(args -> {
        @SuppressWarnings("unchecked") BiConsumer<IngestDocument, Exception> handler = (BiConsumer) args.getArguments()[1];
        handler.accept(null, new RuntimeException());
        return null;
    }).when(processor).execute(eqIndexTypeId(emptyMap()), any());
    final Processor onFailureProcessor = mock(Processor.class);
    doAnswer(args -> {
        IngestDocument ingestDocument = (IngestDocument) args.getArguments()[0];
        @SuppressWarnings("unchecked") BiConsumer<IngestDocument, Exception> handler = (BiConsumer) args.getArguments()[1];
        handler.accept(ingestDocument, null);
        return null;
    }).when(onFailureProcessor).execute(eqIndexTypeId(emptyMap()), any());
    final CompoundProcessor compoundProcessor = new CompoundProcessor(false, Collections.singletonList(processor), Collections.singletonList(new CompoundProcessor(onFailureProcessor)));
    IngestService ingestService = createWithProcessors(Collections.singletonMap("mock", (factories, tag, description, config) -> compoundProcessor));
    PutPipelineRequest putRequest = new PutPipelineRequest("_id", new BytesArray("{\"processors\": [{\"mock\" : {}}]}"), XContentType.JSON);
    // Start empty
    ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).build();
    ClusterState previousClusterState = clusterState;
    clusterState = IngestService.innerPut(putRequest, clusterState);
    ingestService.applyClusterState(new ClusterChangedEvent("", clusterState, previousClusterState));
    final IndexRequest indexRequest = new IndexRequest("_index").id("_id").source(emptyMap()).setPipeline("_id").setFinalPipeline("_none");
    @SuppressWarnings("unchecked") final BiConsumer<Integer, Exception> failureHandler = mock(BiConsumer.class);
    @SuppressWarnings("unchecked") final BiConsumer<Thread, Exception> completionHandler = mock(BiConsumer.class);
    ingestService.executeBulkRequest(1, Collections.singletonList(indexRequest), failureHandler, completionHandler, indexReq -> {
    }, Names.WRITE);
    verify(failureHandler, never()).accept(eq(0), any(IngestProcessorException.class));
    verify(completionHandler, times(1)).accept(Thread.currentThread(), null);
}

Example 37 with PutPipelineRequest

use of org.opensearch.action.ingest.PutPipelineRequest in project OpenSearch by opensearch-project.

the class IngestServiceTests method testExecuteWithDrop.

public void testExecuteWithDrop() {
    Map<String, Processor.Factory> factories = new HashMap<>();
    factories.put("drop", new DropProcessor.Factory());
    factories.put("mock", (processorFactories, tag, description, config) -> new Processor() {

        @Override
        public IngestDocument execute(final IngestDocument ingestDocument) {
            throw new AssertionError("Document should have been dropped but reached this processor");
        }

        @Override
        public String getType() {
            return null;
        }

        @Override
        public String getTag() {
            return null;
        }

        @Override
        public String getDescription() {
            return null;
        }
    });
    IngestService ingestService = createWithProcessors(factories);
    PutPipelineRequest putRequest = new PutPipelineRequest("_id", new BytesArray("{\"processors\": [{\"drop\" : {}}, {\"mock\" : {}}]}"), XContentType.JSON);
    // Start empty
    ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).build();
    ClusterState previousClusterState = clusterState;
    clusterState = IngestService.innerPut(putRequest, clusterState);
    ingestService.applyClusterState(new ClusterChangedEvent("", clusterState, previousClusterState));
    BulkRequest bulkRequest = new BulkRequest();
    final IndexRequest indexRequest1 = new IndexRequest("_index").id("_id1").source(Collections.emptyMap()).setPipeline("_none").setFinalPipeline("_none");
    bulkRequest.add(indexRequest1);
    IndexRequest indexRequest2 = new IndexRequest("_index").id("_id2").source(Collections.emptyMap()).setPipeline("_id").setFinalPipeline("_none");
    bulkRequest.add(indexRequest2);
    @SuppressWarnings("unchecked") final BiConsumer<Integer, Exception> failureHandler = mock(BiConsumer.class);
    @SuppressWarnings("unchecked") final BiConsumer<Thread, Exception> completionHandler = mock(BiConsumer.class);
    @SuppressWarnings("unchecked") final IntConsumer dropHandler = mock(IntConsumer.class);
    ingestService.executeBulkRequest(bulkRequest.numberOfActions(), bulkRequest.requests(), failureHandler, completionHandler, dropHandler, Names.WRITE);
    verify(failureHandler, never()).accept(any(), any());
    verify(completionHandler, times(1)).accept(Thread.currentThread(), null);
    verify(dropHandler, times(1)).accept(1);
}

Example 38 with PutPipelineRequest

use of org.opensearch.action.ingest.PutPipelineRequest in project OpenSearch by opensearch-project.

the class IngestServiceTests method testIngestClusterStateListeners_orderOfExecution.

public void testIngestClusterStateListeners_orderOfExecution() {
    final AtomicInteger counter = new AtomicInteger(0);
    // Ingest cluster state listener state should be invoked first:
    Consumer<ClusterState> ingestClusterStateListener = clusterState -> {
        assertThat(counter.compareAndSet(0, 1), is(true));
    };
    // Processor factory should be invoked secondly after ingest cluster state listener:
    IngestPlugin testPlugin = new IngestPlugin() {

        @Override
        public Map<String, Processor.Factory> getProcessors(Processor.Parameters parameters) {
            return Collections.singletonMap("test", (factories, tag, description, config) -> {
                assertThat(counter.compareAndSet(1, 2), is(true));
                return new FakeProcessor("test", tag, description, ingestDocument -> {
                });
            });
        }
    };
    // Create ingest service:
    Client client = mock(Client.class);
    IngestService ingestService = new IngestService(mock(ClusterService.class), threadPool, null, null, null, Arrays.asList(testPlugin), client);
    ingestService.addIngestClusterStateListener(ingestClusterStateListener);
    // Create pipeline and apply the resulting cluster state, which should update the counter in the right order:
    PutPipelineRequest putRequest = new PutPipelineRequest("_id", new BytesArray("{\"processors\": [{\"test\" : {}}]}"), XContentType.JSON);
    // Start empty
    ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).build();
    ClusterState previousClusterState = clusterState;
    clusterState = IngestService.innerPut(putRequest, clusterState);
    ingestService.applyClusterState(new ClusterChangedEvent("", clusterState, previousClusterState));
    // Sanity check that counter has been updated twice:
    assertThat(counter.get(), equalTo(2));
}

Example 39 with PutPipelineRequest

use of org.opensearch.action.ingest.PutPipelineRequest in project OpenSearch by opensearch-project.

the class IngestClientIT method testBulkWithIngestFailures.

public void testBulkWithIngestFailures() throws Exception {
    createIndex("index");
    BytesReference source = BytesReference.bytes(jsonBuilder().startObject().field("description", "my_pipeline").startArray("processors").startObject().startObject("test").endObject().endObject().endArray().endObject());
    PutPipelineRequest putPipelineRequest = new PutPipelineRequest("_id", source, XContentType.JSON);
    client().admin().cluster().putPipeline(putPipelineRequest).get();
    int numRequests = scaledRandomIntBetween(32, 128);
    BulkRequest bulkRequest = new BulkRequest();
    for (int i = 0; i < numRequests; i++) {
        IndexRequest indexRequest = new IndexRequest("index").id(Integer.toString(i)).setPipeline("_id");
        indexRequest.source(Requests.INDEX_CONTENT_TYPE, "field", "value", "fail", i % 2 == 0);
        bulkRequest.add(indexRequest);
    }
    BulkResponse response = client().bulk(bulkRequest).actionGet();
    assertThat(response.getItems().length, equalTo(bulkRequest.requests().size()));
    for (int i = 0; i < bulkRequest.requests().size(); i++) {
        BulkItemResponse itemResponse = response.getItems()[i];
        if (i % 2 == 0) {
            BulkItemResponse.Failure failure = itemResponse.getFailure();
            OpenSearchException compoundProcessorException = (OpenSearchException) failure.getCause();
            assertThat(compoundProcessorException.getRootCause().getMessage(), equalTo("test processor failed"));
        } else {
            IndexResponse indexResponse = itemResponse.getResponse();
            assertThat("Expected a successful response but found failure [" + itemResponse.getFailure() + "].", itemResponse.isFailed(), is(false));
            assertThat(indexResponse, notNullValue());
            assertThat(indexResponse.getId(), equalTo(Integer.toString(i)));
            assertEquals(DocWriteResponse.Result.CREATED, indexResponse.getResult());
        }
    }
    // cleanup
    AcknowledgedResponse deletePipelineResponse = client().admin().cluster().prepareDeletePipeline("_id").get();
    assertTrue(deletePipelineResponse.isAcknowledged());
}

Example 40 with PutPipelineRequest

use of org.opensearch.action.ingest.PutPipelineRequest in project OpenSearch by opensearch-project.

the class IngestClientIT method testPutWithPipelineFactoryError.

public void testPutWithPipelineFactoryError() throws Exception {
    BytesReference source = BytesReference.bytes(jsonBuilder().startObject().field("description", "my_pipeline").startArray("processors").startObject().startObject("test").field("unused", ":sad_face:").endObject().endObject().endArray().endObject());
    PutPipelineRequest putPipelineRequest = new PutPipelineRequest("_id2", source, XContentType.JSON);
    Exception e = expectThrows(OpenSearchParseException.class, () -> client().admin().cluster().putPipeline(putPipelineRequest).actionGet());
    assertThat(e.getMessage(), equalTo("processor [test] doesn't support one or more provided configuration parameters [unused]"));
    GetPipelineResponse response = client().admin().cluster().prepareGetPipeline("_id2").get();
    assertFalse(response.isFound());
}