Examples with CompoundProcessor org.opensearch.ingest.CompoundProcessor used on opensource projects

Example 1 with CompoundProcessor

use of org.opensearch.ingest.CompoundProcessor in project OpenSearch by opensearch-project.

the class ForEachProcessorTests method testModifyFieldsOutsideArray.

public void testModifyFieldsOutsideArray() throws Exception {
    List<Object> values = new ArrayList<>();
    values.add("string");
    values.add(1);
    values.add(null);
    IngestDocument ingestDocument = new IngestDocument("_index", "_id", null, null, null, Collections.singletonMap("values", values));
    TemplateScript.Factory template = new TestTemplateService.MockTemplateScript.Factory("errors");
    ForEachProcessor processor = new ForEachProcessor("_tag", null, "values", new CompoundProcessor(false, org.opensearch.common.collect.List.of(new UppercaseProcessor("_tag_upper", null, "_ingest._value", false, "_ingest._value")), org.opensearch.common.collect.List.of(new AppendProcessor("_tag", null, template, (model) -> (Collections.singletonList("added")), true))), false);
    processor.execute(ingestDocument, (result, e) -> {
    });
    List<?> result = ingestDocument.getFieldValue("values", List.class);
    assertThat(result.get(0), equalTo("STRING"));
    assertThat(result.get(1), equalTo(1));
    assertThat(result.get(2), equalTo(null));
    List<?> errors = ingestDocument.getFieldValue("errors", List.class);
    assertThat(errors.size(), equalTo(2));
}

Example 2 with CompoundProcessor

use of org.opensearch.ingest.CompoundProcessor in project OpenSearch by opensearch-project.

the class SimulateExecutionService method executeDocument.

void executeDocument(Pipeline pipeline, IngestDocument ingestDocument, boolean verbose, BiConsumer<SimulateDocumentResult, Exception> handler) {
    if (verbose) {
        List<SimulateProcessorResult> processorResultList = new CopyOnWriteArrayList<>();
        CompoundProcessor verbosePipelineProcessor = decorate(pipeline.getCompoundProcessor(), null, processorResultList);
        Pipeline verbosePipeline = new Pipeline(pipeline.getId(), pipeline.getDescription(), pipeline.getVersion(), verbosePipelineProcessor);
        ingestDocument.executePipeline(verbosePipeline, (result, e) -> {
            handler.accept(new SimulateDocumentVerboseResult(processorResultList), e);
        });
    } else {
        ingestDocument.executePipeline(pipeline, (result, e) -> {
            if (e == null) {
                handler.accept(new SimulateDocumentBaseResult(result), null);
            } else {
                handler.accept(new SimulateDocumentBaseResult(e), null);
            }
        });
    }
}

Example 3 with CompoundProcessor

use of org.opensearch.ingest.CompoundProcessor in project OpenSearch by opensearch-project.

the class SimulateExecutionServiceTests method testDropDocumentVerboseExtraProcessor.

public void testDropDocumentVerboseExtraProcessor() throws Exception {
    TestProcessor processor1 = new TestProcessor(ingestDocument -> ingestDocument.setFieldValue("field1", "value"));
    Processor processor2 = new DropProcessor.Factory().create(Collections.emptyMap(), null, null, Collections.emptyMap());
    TestProcessor processor3 = new TestProcessor(ingestDocument -> ingestDocument.setFieldValue("field2", "value"));
    Pipeline pipeline = new Pipeline("_id", "_description", version, new CompoundProcessor(processor1, processor2, processor3));
    CountDownLatch latch = new CountDownLatch(1);
    AtomicReference<SimulateDocumentResult> holder = new AtomicReference<>();
    executionService.executeDocument(pipeline, ingestDocument, true, (r, e) -> {
        holder.set(r);
        latch.countDown();
    });
    latch.await();
    SimulateDocumentResult actualItemResponse = holder.get();
    assertThat(processor1.getInvokedCounter(), equalTo(1));
    assertThat(processor3.getInvokedCounter(), equalTo(0));
    assertThat(actualItemResponse, instanceOf(SimulateDocumentVerboseResult.class));
    SimulateDocumentVerboseResult verboseResult = (SimulateDocumentVerboseResult) actualItemResponse;
    assertThat(verboseResult.getProcessorResults().size(), equalTo(2));
    assertThat(verboseResult.getProcessorResults().get(0).getIngestDocument(), notNullValue());
    assertThat(verboseResult.getProcessorResults().get(0).getFailure(), nullValue());
    assertThat(verboseResult.getProcessorResults().get(1).getIngestDocument(), nullValue());
    assertThat(verboseResult.getProcessorResults().get(1).getFailure(), nullValue());
}

Example 4 with CompoundProcessor

use of org.opensearch.ingest.CompoundProcessor in project OpenSearch by opensearch-project.

the class SimulateExecutionServiceTests method testExecuteVerboseItemWithOnFailure.

public void testExecuteVerboseItemWithOnFailure() throws Exception {
    TestProcessor processor1 = new TestProcessor("processor_0", "mock", null, new RuntimeException("processor failed"));
    TestProcessor processor2 = new TestProcessor("processor_1", "mock", null, ingestDocument -> {
    });
    TestProcessor processor3 = new TestProcessor("processor_2", "mock", null, ingestDocument -> {
    });
    Pipeline pipeline = new Pipeline("_id", "_description", version, new CompoundProcessor(new CompoundProcessor(false, Collections.singletonList(processor1), Collections.singletonList(processor2)), processor3));
    CountDownLatch latch = new CountDownLatch(1);
    AtomicReference<SimulateDocumentResult> holder = new AtomicReference<>();
    executionService.executeDocument(pipeline, ingestDocument, true, (r, e) -> {
        holder.set(r);
        latch.countDown();
    });
    latch.await();
    SimulateDocumentResult actualItemResponse = holder.get();
    assertThat(processor1.getInvokedCounter(), equalTo(1));
    assertThat(processor2.getInvokedCounter(), equalTo(1));
    assertThat(actualItemResponse, instanceOf(SimulateDocumentVerboseResult.class));
    SimulateDocumentVerboseResult simulateDocumentVerboseResult = (SimulateDocumentVerboseResult) actualItemResponse;
    assertThat(simulateDocumentVerboseResult.getProcessorResults().size(), equalTo(3));
    assertThat(simulateDocumentVerboseResult.getProcessorResults().get(0).getProcessorTag(), equalTo("processor_0"));
    assertThat(simulateDocumentVerboseResult.getProcessorResults().get(0).getIngestDocument(), nullValue());
    assertThat(simulateDocumentVerboseResult.getProcessorResults().get(0).getFailure(), instanceOf(RuntimeException.class));
    RuntimeException runtimeException = (RuntimeException) simulateDocumentVerboseResult.getProcessorResults().get(0).getFailure();
    assertThat(runtimeException.getMessage(), equalTo("processor failed"));
    assertThat(simulateDocumentVerboseResult.getProcessorResults().get(1).getProcessorTag(), equalTo("processor_1"));
    assertThat(simulateDocumentVerboseResult.getProcessorResults().get(1).getIngestDocument(), not(sameInstance(ingestDocument)));
    IngestDocument ingestDocumentWithOnFailureMetadata = new IngestDocument(ingestDocument);
    Map<String, Object> metadata = ingestDocumentWithOnFailureMetadata.getIngestMetadata();
    metadata.put(CompoundProcessor.ON_FAILURE_PROCESSOR_TYPE_FIELD, "mock");
    metadata.put(CompoundProcessor.ON_FAILURE_PROCESSOR_TAG_FIELD, "processor_0");
    metadata.put(CompoundProcessor.ON_FAILURE_MESSAGE_FIELD, "processor failed");
    assertVerboseResult(simulateDocumentVerboseResult.getProcessorResults().get(1), pipeline.getId(), ingestDocumentWithOnFailureMetadata);
    assertThat(simulateDocumentVerboseResult.getProcessorResults().get(1).getFailure(), nullValue());
    assertThat(simulateDocumentVerboseResult.getProcessorResults().get(2).getProcessorTag(), equalTo("processor_2"));
    assertVerboseResult(simulateDocumentVerboseResult.getProcessorResults().get(2), pipeline.getId(), ingestDocument);
    assertThat(simulateDocumentVerboseResult.getProcessorResults().get(2).getFailure(), nullValue());
}

Example 5 with CompoundProcessor

use of org.opensearch.ingest.CompoundProcessor in project OpenSearch by opensearch-project.

the class SimulateExecutionServiceTests method testExecuteVerboseItemExceptionWithoutOnFailure.

public void testExecuteVerboseItemExceptionWithoutOnFailure() throws Exception {
    TestProcessor processor1 = new TestProcessor("processor_0", "mock", null, ingestDocument -> {
    });
    TestProcessor processor2 = new TestProcessor("processor_1", "mock", null, new RuntimeException("processor failed"));
    TestProcessor processor3 = new TestProcessor("processor_2", "mock", null, ingestDocument -> {
    });
    Pipeline pipeline = new Pipeline("_id", "_description", version, new CompoundProcessor(processor1, processor2, processor3));
    CountDownLatch latch = new CountDownLatch(1);
    AtomicReference<SimulateDocumentResult> holder = new AtomicReference<>();
    executionService.executeDocument(pipeline, ingestDocument, true, (r, e) -> {
        holder.set(r);
        latch.countDown();
    });
    latch.await();
    SimulateDocumentResult actualItemResponse = holder.get();
    assertThat(processor1.getInvokedCounter(), equalTo(1));
    assertThat(processor2.getInvokedCounter(), equalTo(1));
    assertThat(processor3.getInvokedCounter(), equalTo(0));
    assertThat(actualItemResponse, instanceOf(SimulateDocumentVerboseResult.class));
    SimulateDocumentVerboseResult simulateDocumentVerboseResult = (SimulateDocumentVerboseResult) actualItemResponse;
    assertThat(simulateDocumentVerboseResult.getProcessorResults().size(), equalTo(2));
    assertThat(simulateDocumentVerboseResult.getProcessorResults().get(0).getProcessorTag(), equalTo("processor_0"));
    assertThat(simulateDocumentVerboseResult.getProcessorResults().get(0).getFailure(), nullValue());
    assertVerboseResult(simulateDocumentVerboseResult.getProcessorResults().get(0), pipeline.getId(), ingestDocument);
    assertThat(simulateDocumentVerboseResult.getProcessorResults().get(1).getProcessorTag(), equalTo("processor_1"));
    assertThat(simulateDocumentVerboseResult.getProcessorResults().get(1).getIngestDocument(), nullValue());
    assertThat(simulateDocumentVerboseResult.getProcessorResults().get(1).getFailure(), instanceOf(RuntimeException.class));
    RuntimeException runtimeException = (RuntimeException) simulateDocumentVerboseResult.getProcessorResults().get(1).getFailure();
    assertThat(runtimeException.getMessage(), equalTo("processor failed"));
}