Examples with SimulateProcessorResult org.opensearch.action.ingest.SimulateProcessorResult used on opensource projects

Example 1 with SimulateProcessorResult

use of org.opensearch.action.ingest.SimulateProcessorResult in project OpenSearch by opensearch-project.

the class TrackingResultProcessor method execute.

@Override
public void execute(IngestDocument ingestDocument, BiConsumer<IngestDocument, Exception> handler) {
    Tuple<String, Boolean> conditionalWithResult;
    if (conditionalProcessor != null) {
        if (conditionalProcessor.evaluate(ingestDocument) == false) {
            conditionalWithResult = new Tuple<>(conditionalProcessor.getCondition(), Boolean.FALSE);
            processorResultList.add(new SimulateProcessorResult(actualProcessor.getType(), actualProcessor.getTag(), actualProcessor.getDescription(), conditionalWithResult));
            handler.accept(ingestDocument, null);
            return;
        } else {
            conditionalWithResult = new Tuple<>(conditionalProcessor.getCondition(), Boolean.TRUE);
        }
    } else {
        // no condition
        conditionalWithResult = null;
    }
    if (actualProcessor instanceof PipelineProcessor) {
        PipelineProcessor pipelineProcessor = ((PipelineProcessor) actualProcessor);
        Pipeline pipeline = pipelineProcessor.getPipeline(ingestDocument);
        // runtime check for cycles against a copy of the document. This is needed to properly handle conditionals around pipelines
        IngestDocument ingestDocumentCopy = new IngestDocument(ingestDocument);
        Pipeline pipelineToCall = pipelineProcessor.getPipeline(ingestDocument);
        if (pipelineToCall == null) {
            throw new IllegalArgumentException("Pipeline processor configured for non-existent pipeline [" + pipelineProcessor.getPipelineToCallName(ingestDocument) + ']');
        }
        ingestDocumentCopy.executePipeline(pipelineToCall, (result, e) -> {
            // special handling for pipeline cycle errors
            if (e instanceof OpenSearchException && e.getCause() instanceof IllegalStateException && e.getCause().getMessage().startsWith(PIPELINE_CYCLE_ERROR_MESSAGE)) {
                if (ignoreFailure) {
                    processorResultList.add(new SimulateProcessorResult(pipelineProcessor.getType(), pipelineProcessor.getTag(), pipelineProcessor.getDescription(), new IngestDocument(ingestDocument), e, conditionalWithResult));
                } else {
                    processorResultList.add(new SimulateProcessorResult(pipelineProcessor.getType(), pipelineProcessor.getTag(), pipelineProcessor.getDescription(), e, conditionalWithResult));
                }
                handler.accept(null, e);
            } else {
                // now that we know that there are no cycles between pipelines, decorate the processors for this pipeline and
                // execute it
                CompoundProcessor verbosePipelineProcessor = decorate(pipeline.getCompoundProcessor(), null, processorResultList);
                // add the pipeline process to the results
                processorResultList.add(new SimulateProcessorResult(actualProcessor.getType(), actualProcessor.getTag(), actualProcessor.getDescription(), conditionalWithResult));
                Pipeline verbosePipeline = new Pipeline(pipeline.getId(), pipeline.getDescription(), pipeline.getVersion(), verbosePipelineProcessor);
                ingestDocument.executePipeline(verbosePipeline, handler);
            }
        });
        return;
    }
    actualProcessor.execute(ingestDocument, (result, e) -> {
        if (e != null) {
            if (ignoreFailure) {
                processorResultList.add(new SimulateProcessorResult(actualProcessor.getType(), actualProcessor.getTag(), actualProcessor.getDescription(), new IngestDocument(ingestDocument), e, conditionalWithResult));
            } else {
                processorResultList.add(new SimulateProcessorResult(actualProcessor.getType(), actualProcessor.getTag(), actualProcessor.getDescription(), e, conditionalWithResult));
            }
            handler.accept(null, e);
        } else {
            if (result != null) {
                processorResultList.add(new SimulateProcessorResult(actualProcessor.getType(), actualProcessor.getTag(), actualProcessor.getDescription(), new IngestDocument(ingestDocument), conditionalWithResult));
                handler.accept(result, null);
            } else {
                processorResultList.add(new SimulateProcessorResult(actualProcessor.getType(), actualProcessor.getTag(), actualProcessor.getDescription(), conditionalWithResult));
                handler.accept(null, null);
            }
        }
    });
}

Example 2 with SimulateProcessorResult

use of org.opensearch.action.ingest.SimulateProcessorResult in project OpenSearch by opensearch-project.

the class TrackingResultProcessorTests method testActualPipelineProcessor.

public void testActualPipelineProcessor() throws Exception {
    String pipelineId = "pipeline1";
    IngestService ingestService = createIngestService();
    Map<String, Object> pipelineConfig = new HashMap<>();
    pipelineConfig.put("name", pipelineId);
    PipelineProcessor.Factory factory = new PipelineProcessor.Factory(ingestService);
    String key1 = randomAlphaOfLength(10);
    String key2 = randomAlphaOfLength(10);
    String key3 = randomAlphaOfLength(10);
    Pipeline pipeline = new Pipeline(pipelineId, null, null, new CompoundProcessor(new TestProcessor(ingestDocument -> {
        ingestDocument.setFieldValue(key1, randomInt());
    }), new TestProcessor(ingestDocument -> {
        ingestDocument.setFieldValue(key2, randomInt());
    }), new TestProcessor(ingestDocument -> {
        ingestDocument.setFieldValue(key3, randomInt());
    })));
    when(ingestService.getPipeline(pipelineId)).thenReturn(pipeline);
    PipelineProcessor pipelineProcessor = factory.create(Collections.emptyMap(), null, null, pipelineConfig);
    CompoundProcessor actualProcessor = new CompoundProcessor(pipelineProcessor);
    CompoundProcessor trackingProcessor = decorate(actualProcessor, null, resultList);
    trackingProcessor.execute(ingestDocument, (result, e) -> {
    });
    SimulateProcessorResult expectedResult = new SimulateProcessorResult(actualProcessor.getType(), actualProcessor.getTag(), actualProcessor.getDescription(), ingestDocument, null);
    expectedResult.getIngestDocument().getIngestMetadata().put("pipeline", pipelineId);
    verify(ingestService, Mockito.atLeast(1)).getPipeline(pipelineId);
    assertThat(resultList.size(), equalTo(4));
    assertNull(resultList.get(0).getConditionalWithResult());
    assertThat(resultList.get(0).getType(), equalTo("pipeline"));
    assertTrue(resultList.get(1).getIngestDocument().hasField(key1));
    assertFalse(resultList.get(1).getIngestDocument().hasField(key2));
    assertFalse(resultList.get(1).getIngestDocument().hasField(key3));
    assertTrue(resultList.get(2).getIngestDocument().hasField(key1));
    assertTrue(resultList.get(2).getIngestDocument().hasField(key2));
    assertFalse(resultList.get(2).getIngestDocument().hasField(key3));
    assertThat(resultList.get(3).getIngestDocument(), equalTo(expectedResult.getIngestDocument()));
    assertThat(resultList.get(3).getFailure(), nullValue());
    assertThat(resultList.get(3).getProcessorTag(), nullValue());
}

Example 3 with SimulateProcessorResult

use of org.opensearch.action.ingest.SimulateProcessorResult in project OpenSearch by opensearch-project.

the class TrackingResultProcessorTests method testActualCompoundProcessorWithFalseConditional.

public void testActualCompoundProcessorWithFalseConditional() throws Exception {
    String key1 = randomAlphaOfLength(10);
    String key2 = randomAlphaOfLength(10);
    String key3 = randomAlphaOfLength(10);
    String scriptName = "conditionalScript";
    ScriptService scriptService = new ScriptService(Settings.builder().build(), Collections.singletonMap(Script.DEFAULT_SCRIPT_LANG, new MockScriptEngine(Script.DEFAULT_SCRIPT_LANG, Collections.singletonMap(scriptName, ctx -> false), Collections.emptyMap())), new HashMap<>(ScriptModule.CORE_CONTEXTS));
    CompoundProcessor compoundProcessor = new CompoundProcessor(new TestProcessor(ingestDocument -> {
        ingestDocument.setFieldValue(key1, randomInt());
    }), new ConditionalProcessor(randomAlphaOfLength(10), null, new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, scriptName, Collections.emptyMap()), scriptService, new TestProcessor(ingestDocument -> {
        ingestDocument.setFieldValue(key2, randomInt());
    })), new TestProcessor(ingestDocument -> {
        ingestDocument.setFieldValue(key3, randomInt());
    }));
    CompoundProcessor trackingProcessor = decorate(compoundProcessor, null, resultList);
    trackingProcessor.execute(ingestDocument, (result, e) -> {
    });
    SimulateProcessorResult expectedResult = new SimulateProcessorResult(compoundProcessor.getType(), compoundProcessor.getTag(), compoundProcessor.getDescription(), ingestDocument, null);
    assertThat(resultList.size(), equalTo(3));
    assertTrue(resultList.get(0).getIngestDocument().hasField(key1));
    assertFalse(resultList.get(0).getIngestDocument().hasField(key2));
    assertFalse(resultList.get(0).getIngestDocument().hasField(key3));
    assertThat(resultList.get(1).getConditionalWithResult().v1(), equalTo(scriptName));
    assertThat(resultList.get(1).getConditionalWithResult().v2(), is(Boolean.FALSE));
    assertTrue(resultList.get(2).getIngestDocument().hasField(key1));
    assertFalse(resultList.get(2).getIngestDocument().hasField(key2));
    assertTrue(resultList.get(2).getIngestDocument().hasField(key3));
    assertThat(resultList.get(2).getIngestDocument(), equalTo(expectedResult.getIngestDocument()));
    assertThat(resultList.get(2).getFailure(), nullValue());
    assertThat(resultList.get(2).getProcessorTag(), nullValue());
}

Example 4 with SimulateProcessorResult

use of org.opensearch.action.ingest.SimulateProcessorResult in project OpenSearch by opensearch-project.

the class TrackingResultProcessorTests method testActualPipelineProcessorRepeatedInvocation.

public void testActualPipelineProcessorRepeatedInvocation() throws Exception {
    String pipelineId = "pipeline1";
    IngestService ingestService = createIngestService();
    Map<String, Object> pipelineConfig = new HashMap<>();
    pipelineConfig.put("name", pipelineId);
    PipelineProcessor.Factory factory = new PipelineProcessor.Factory(ingestService);
    String key1 = randomAlphaOfLength(10);
    PipelineProcessor pipelineProcessor = factory.create(Collections.emptyMap(), null, null, pipelineConfig);
    Pipeline pipeline = new Pipeline(pipelineId, null, null, new CompoundProcessor(new TestProcessor(ingestDocument -> {
        ingestDocument.setFieldValue(key1, randomInt());
    })));
    when(ingestService.getPipeline(pipelineId)).thenReturn(pipeline);
    // calls the same pipeline twice
    CompoundProcessor actualProcessor = new CompoundProcessor(pipelineProcessor, pipelineProcessor);
    CompoundProcessor trackingProcessor = decorate(actualProcessor, null, resultList);
    trackingProcessor.execute(ingestDocument, (result, e) -> {
    });
    SimulateProcessorResult expectedResult = new SimulateProcessorResult(actualProcessor.getType(), actualProcessor.getTag(), actualProcessor.getDescription(), ingestDocument, null);
    expectedResult.getIngestDocument().getIngestMetadata().put("pipeline", pipelineId);
    verify(ingestService, Mockito.atLeast(2)).getPipeline(pipelineId);
    assertThat(resultList.size(), equalTo(4));
    assertNull(resultList.get(0).getConditionalWithResult());
    assertThat(resultList.get(0).getType(), equalTo("pipeline"));
    assertThat(resultList.get(1).getIngestDocument(), not(equalTo(expectedResult.getIngestDocument())));
    assertThat(resultList.get(1).getFailure(), nullValue());
    assertThat(resultList.get(1).getProcessorTag(), nullValue());
    assertNull(resultList.get(2).getConditionalWithResult());
    assertThat(resultList.get(2).getType(), equalTo("pipeline"));
    assertThat(resultList.get(3).getIngestDocument(), equalTo(expectedResult.getIngestDocument()));
    assertThat(resultList.get(3).getFailure(), nullValue());
    assertThat(resultList.get(3).getProcessorTag(), nullValue());
    // each invocation updates key1 with a random int
    assertNotEquals(resultList.get(1).getIngestDocument().getSourceAndMetadata().get(key1), resultList.get(3).getIngestDocument().getSourceAndMetadata().get(key1));
}

Example 5 with SimulateProcessorResult

use of org.opensearch.action.ingest.SimulateProcessorResult in project OpenSearch by opensearch-project.

the class TrackingResultProcessorTests method testActualPipelineProcessorWithFalseConditional.

public void testActualPipelineProcessorWithFalseConditional() throws Exception {
    String pipelineId1 = "pipeline1";
    String pipelineId2 = "pipeline2";
    IngestService ingestService = createIngestService();
    Map<String, Object> pipelineConfig0 = new HashMap<>();
    pipelineConfig0.put("name", pipelineId1);
    Map<String, Object> pipelineConfig1 = new HashMap<>();
    pipelineConfig1.put("name", pipelineId1);
    Map<String, Object> pipelineConfig2 = new HashMap<>();
    pipelineConfig2.put("name", pipelineId2);
    PipelineProcessor.Factory factory = new PipelineProcessor.Factory(ingestService);
    String key1 = randomAlphaOfLength(10);
    String key2 = randomAlphaOfLength(10);
    String key3 = randomAlphaOfLength(10);
    String scriptName = "conditionalScript";
    ScriptService scriptService = new ScriptService(Settings.builder().build(), Collections.singletonMap(Script.DEFAULT_SCRIPT_LANG, new MockScriptEngine(Script.DEFAULT_SCRIPT_LANG, Collections.singletonMap(scriptName, ctx -> false), Collections.emptyMap())), new HashMap<>(ScriptModule.CORE_CONTEXTS));
    Pipeline pipeline1 = new Pipeline(pipelineId1, null, null, new CompoundProcessor(new TestProcessor(ingestDocument -> {
        ingestDocument.setFieldValue(key1, randomInt());
    }), new ConditionalProcessor(randomAlphaOfLength(10), null, new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, scriptName, Collections.emptyMap()), scriptService, factory.create(Collections.emptyMap(), null, null, pipelineConfig2)), new TestProcessor(ingestDocument -> {
        ingestDocument.setFieldValue(key3, randomInt());
    })));
    Pipeline pipeline2 = new Pipeline(pipelineId2, null, null, new CompoundProcessor(new TestProcessor(ingestDocument -> {
        ingestDocument.setFieldValue(key2, randomInt());
    })));
    when(ingestService.getPipeline(pipelineId1)).thenReturn(pipeline1);
    when(ingestService.getPipeline(pipelineId2)).thenReturn(pipeline2);
    PipelineProcessor pipelineProcessor = factory.create(Collections.emptyMap(), null, null, pipelineConfig0);
    CompoundProcessor actualProcessor = new CompoundProcessor(pipelineProcessor);
    CompoundProcessor trackingProcessor = decorate(actualProcessor, null, resultList);
    trackingProcessor.execute(ingestDocument, (result, e) -> {
    });
    SimulateProcessorResult expectedResult = new SimulateProcessorResult(actualProcessor.getType(), actualProcessor.getTag(), actualProcessor.getDescription(), ingestDocument, null);
    expectedResult.getIngestDocument().getIngestMetadata().put("pipeline", pipelineId1);
    verify(ingestService, Mockito.atLeast(1)).getPipeline(pipelineId1);
    verify(ingestService, Mockito.never()).getPipeline(pipelineId2);
    assertThat(resultList.size(), equalTo(4));
    assertNull(resultList.get(0).getConditionalWithResult());
    assertThat(resultList.get(0).getType(), equalTo("pipeline"));
    assertTrue(resultList.get(1).getIngestDocument().hasField(key1));
    assertFalse(resultList.get(1).getIngestDocument().hasField(key2));
    assertFalse(resultList.get(1).getIngestDocument().hasField(key3));
    assertThat(resultList.get(2).getConditionalWithResult().v1(), equalTo(scriptName));
    assertThat(resultList.get(2).getConditionalWithResult().v2(), is(Boolean.FALSE));
    assertThat(resultList.get(3).getIngestDocument(), equalTo(expectedResult.getIngestDocument()));
    assertThat(resultList.get(3).getFailure(), nullValue());
    assertThat(resultList.get(3).getProcessorTag(), nullValue());
}