Examples with Pipeline org.opensearch.ingest.Pipeline used on opensource projects

Example 1 with Pipeline

use of org.opensearch.ingest.Pipeline in project OpenSearch by opensearch-project.

the class SimulateExecutionService method executeDocument.

void executeDocument(Pipeline pipeline, IngestDocument ingestDocument, boolean verbose, BiConsumer<SimulateDocumentResult, Exception> handler) {
    if (verbose) {
        List<SimulateProcessorResult> processorResultList = new CopyOnWriteArrayList<>();
        CompoundProcessor verbosePipelineProcessor = decorate(pipeline.getCompoundProcessor(), null, processorResultList);
        Pipeline verbosePipeline = new Pipeline(pipeline.getId(), pipeline.getDescription(), pipeline.getVersion(), verbosePipelineProcessor);
        ingestDocument.executePipeline(verbosePipeline, (result, e) -> {
            handler.accept(new SimulateDocumentVerboseResult(processorResultList), e);
        });
    } else {
        ingestDocument.executePipeline(pipeline, (result, e) -> {
            if (e == null) {
                handler.accept(new SimulateDocumentBaseResult(result), null);
            } else {
                handler.accept(new SimulateDocumentBaseResult(e), null);
            }
        });
    }
}

Example 2 with Pipeline

use of org.opensearch.ingest.Pipeline in project OpenSearch by opensearch-project.

the class SimulateExecutionServiceTests method testDropDocumentVerboseExtraProcessor.

public void testDropDocumentVerboseExtraProcessor() throws Exception {
    TestProcessor processor1 = new TestProcessor(ingestDocument -> ingestDocument.setFieldValue("field1", "value"));
    Processor processor2 = new DropProcessor.Factory().create(Collections.emptyMap(), null, null, Collections.emptyMap());
    TestProcessor processor3 = new TestProcessor(ingestDocument -> ingestDocument.setFieldValue("field2", "value"));
    Pipeline pipeline = new Pipeline("_id", "_description", version, new CompoundProcessor(processor1, processor2, processor3));
    CountDownLatch latch = new CountDownLatch(1);
    AtomicReference<SimulateDocumentResult> holder = new AtomicReference<>();
    executionService.executeDocument(pipeline, ingestDocument, true, (r, e) -> {
        holder.set(r);
        latch.countDown();
    });
    latch.await();
    SimulateDocumentResult actualItemResponse = holder.get();
    assertThat(processor1.getInvokedCounter(), equalTo(1));
    assertThat(processor3.getInvokedCounter(), equalTo(0));
    assertThat(actualItemResponse, instanceOf(SimulateDocumentVerboseResult.class));
    SimulateDocumentVerboseResult verboseResult = (SimulateDocumentVerboseResult) actualItemResponse;
    assertThat(verboseResult.getProcessorResults().size(), equalTo(2));
    assertThat(verboseResult.getProcessorResults().get(0).getIngestDocument(), notNullValue());
    assertThat(verboseResult.getProcessorResults().get(0).getFailure(), nullValue());
    assertThat(verboseResult.getProcessorResults().get(1).getIngestDocument(), nullValue());
    assertThat(verboseResult.getProcessorResults().get(1).getFailure(), nullValue());
}

Example 3 with Pipeline

use of org.opensearch.ingest.Pipeline in project OpenSearch by opensearch-project.

the class SimulateExecutionServiceTests method testExecuteVerboseItemWithOnFailure.

public void testExecuteVerboseItemWithOnFailure() throws Exception {
    TestProcessor processor1 = new TestProcessor("processor_0", "mock", null, new RuntimeException("processor failed"));
    TestProcessor processor2 = new TestProcessor("processor_1", "mock", null, ingestDocument -> {
    });
    TestProcessor processor3 = new TestProcessor("processor_2", "mock", null, ingestDocument -> {
    });
    Pipeline pipeline = new Pipeline("_id", "_description", version, new CompoundProcessor(new CompoundProcessor(false, Collections.singletonList(processor1), Collections.singletonList(processor2)), processor3));
    CountDownLatch latch = new CountDownLatch(1);
    AtomicReference<SimulateDocumentResult> holder = new AtomicReference<>();
    executionService.executeDocument(pipeline, ingestDocument, true, (r, e) -> {
        holder.set(r);
        latch.countDown();
    });
    latch.await();
    SimulateDocumentResult actualItemResponse = holder.get();
    assertThat(processor1.getInvokedCounter(), equalTo(1));
    assertThat(processor2.getInvokedCounter(), equalTo(1));
    assertThat(actualItemResponse, instanceOf(SimulateDocumentVerboseResult.class));
    SimulateDocumentVerboseResult simulateDocumentVerboseResult = (SimulateDocumentVerboseResult) actualItemResponse;
    assertThat(simulateDocumentVerboseResult.getProcessorResults().size(), equalTo(3));
    assertThat(simulateDocumentVerboseResult.getProcessorResults().get(0).getProcessorTag(), equalTo("processor_0"));
    assertThat(simulateDocumentVerboseResult.getProcessorResults().get(0).getIngestDocument(), nullValue());
    assertThat(simulateDocumentVerboseResult.getProcessorResults().get(0).getFailure(), instanceOf(RuntimeException.class));
    RuntimeException runtimeException = (RuntimeException) simulateDocumentVerboseResult.getProcessorResults().get(0).getFailure();
    assertThat(runtimeException.getMessage(), equalTo("processor failed"));
    assertThat(simulateDocumentVerboseResult.getProcessorResults().get(1).getProcessorTag(), equalTo("processor_1"));
    assertThat(simulateDocumentVerboseResult.getProcessorResults().get(1).getIngestDocument(), not(sameInstance(ingestDocument)));
    IngestDocument ingestDocumentWithOnFailureMetadata = new IngestDocument(ingestDocument);
    Map<String, Object> metadata = ingestDocumentWithOnFailureMetadata.getIngestMetadata();
    metadata.put(CompoundProcessor.ON_FAILURE_PROCESSOR_TYPE_FIELD, "mock");
    metadata.put(CompoundProcessor.ON_FAILURE_PROCESSOR_TAG_FIELD, "processor_0");
    metadata.put(CompoundProcessor.ON_FAILURE_MESSAGE_FIELD, "processor failed");
    assertVerboseResult(simulateDocumentVerboseResult.getProcessorResults().get(1), pipeline.getId(), ingestDocumentWithOnFailureMetadata);
    assertThat(simulateDocumentVerboseResult.getProcessorResults().get(1).getFailure(), nullValue());
    assertThat(simulateDocumentVerboseResult.getProcessorResults().get(2).getProcessorTag(), equalTo("processor_2"));
    assertVerboseResult(simulateDocumentVerboseResult.getProcessorResults().get(2), pipeline.getId(), ingestDocument);
    assertThat(simulateDocumentVerboseResult.getProcessorResults().get(2).getFailure(), nullValue());
}

Example 4 with Pipeline

use of org.opensearch.ingest.Pipeline in project OpenSearch by opensearch-project.

the class SimulateExecutionServiceTests method testExecuteVerboseItemExceptionWithoutOnFailure.

public void testExecuteVerboseItemExceptionWithoutOnFailure() throws Exception {
    TestProcessor processor1 = new TestProcessor("processor_0", "mock", null, ingestDocument -> {
    });
    TestProcessor processor2 = new TestProcessor("processor_1", "mock", null, new RuntimeException("processor failed"));
    TestProcessor processor3 = new TestProcessor("processor_2", "mock", null, ingestDocument -> {
    });
    Pipeline pipeline = new Pipeline("_id", "_description", version, new CompoundProcessor(processor1, processor2, processor3));
    CountDownLatch latch = new CountDownLatch(1);
    AtomicReference<SimulateDocumentResult> holder = new AtomicReference<>();
    executionService.executeDocument(pipeline, ingestDocument, true, (r, e) -> {
        holder.set(r);
        latch.countDown();
    });
    latch.await();
    SimulateDocumentResult actualItemResponse = holder.get();
    assertThat(processor1.getInvokedCounter(), equalTo(1));
    assertThat(processor2.getInvokedCounter(), equalTo(1));
    assertThat(processor3.getInvokedCounter(), equalTo(0));
    assertThat(actualItemResponse, instanceOf(SimulateDocumentVerboseResult.class));
    SimulateDocumentVerboseResult simulateDocumentVerboseResult = (SimulateDocumentVerboseResult) actualItemResponse;
    assertThat(simulateDocumentVerboseResult.getProcessorResults().size(), equalTo(2));
    assertThat(simulateDocumentVerboseResult.getProcessorResults().get(0).getProcessorTag(), equalTo("processor_0"));
    assertThat(simulateDocumentVerboseResult.getProcessorResults().get(0).getFailure(), nullValue());
    assertVerboseResult(simulateDocumentVerboseResult.getProcessorResults().get(0), pipeline.getId(), ingestDocument);
    assertThat(simulateDocumentVerboseResult.getProcessorResults().get(1).getProcessorTag(), equalTo("processor_1"));
    assertThat(simulateDocumentVerboseResult.getProcessorResults().get(1).getIngestDocument(), nullValue());
    assertThat(simulateDocumentVerboseResult.getProcessorResults().get(1).getFailure(), instanceOf(RuntimeException.class));
    RuntimeException runtimeException = (RuntimeException) simulateDocumentVerboseResult.getProcessorResults().get(1).getFailure();
    assertThat(runtimeException.getMessage(), equalTo("processor failed"));
}

Example 5 with Pipeline

use of org.opensearch.ingest.Pipeline in project OpenSearch by opensearch-project.

the class SimulateExecutionServiceTests method testDropDocument.

public void testDropDocument() throws Exception {
    TestProcessor processor1 = new TestProcessor(ingestDocument -> ingestDocument.setFieldValue("field", "value"));
    Processor processor2 = new DropProcessor.Factory().create(Collections.emptyMap(), null, null, Collections.emptyMap());
    Pipeline pipeline = new Pipeline("_id", "_description", version, new CompoundProcessor(processor1, processor2));
    CountDownLatch latch = new CountDownLatch(1);
    AtomicReference<SimulateDocumentResult> holder = new AtomicReference<>();
    executionService.executeDocument(pipeline, ingestDocument, false, (r, e) -> {
        holder.set(r);
        latch.countDown();
    });
    latch.await();
    SimulateDocumentResult actualItemResponse = holder.get();
    assertThat(processor1.getInvokedCounter(), equalTo(1));
    assertThat(actualItemResponse, instanceOf(SimulateDocumentBaseResult.class));
    SimulateDocumentBaseResult simulateDocumentBaseResult = (SimulateDocumentBaseResult) actualItemResponse;
    assertThat(simulateDocumentBaseResult.getIngestDocument(), nullValue());
    assertThat(simulateDocumentBaseResult.getFailure(), nullValue());
}