Examples with IngestDocument org.elasticsearch.ingest.IngestDocument used on opensource projects

Example 81 with IngestDocument

use of org.elasticsearch.ingest.IngestDocument in project elasticsearch by elastic.

the class SimulateExecutionServiceTests method testExecuteVerboseItem.

public void testExecuteVerboseItem() throws Exception {
    TestProcessor processor = new TestProcessor("test-id", "mock", ingestDocument -> {
    });
    Pipeline pipeline = new Pipeline("_id", "_description", version, new CompoundProcessor(processor, processor));
    SimulateDocumentResult actualItemResponse = executionService.executeDocument(pipeline, ingestDocument, true);
    assertThat(processor.getInvokedCounter(), equalTo(2));
    assertThat(actualItemResponse, instanceOf(SimulateDocumentVerboseResult.class));
    SimulateDocumentVerboseResult simulateDocumentVerboseResult = (SimulateDocumentVerboseResult) actualItemResponse;
    assertThat(simulateDocumentVerboseResult.getProcessorResults().size(), equalTo(2));
    assertThat(simulateDocumentVerboseResult.getProcessorResults().get(0).getProcessorTag(), equalTo("test-id"));
    IngestDocument firstProcessorIngestDocument = simulateDocumentVerboseResult.getProcessorResults().get(0).getIngestDocument();
    assertThat(firstProcessorIngestDocument, not(sameInstance(this.ingestDocument)));
    assertIngestDocument(firstProcessorIngestDocument, this.ingestDocument);
    assertThat(firstProcessorIngestDocument.getSourceAndMetadata(), not(sameInstance(this.ingestDocument.getSourceAndMetadata())));
    assertThat(simulateDocumentVerboseResult.getProcessorResults().get(0).getFailure(), nullValue());
    assertThat(simulateDocumentVerboseResult.getProcessorResults().get(1).getProcessorTag(), equalTo("test-id"));
    IngestDocument secondProcessorIngestDocument = simulateDocumentVerboseResult.getProcessorResults().get(1).getIngestDocument();
    assertThat(secondProcessorIngestDocument, not(sameInstance(this.ingestDocument)));
    assertIngestDocument(secondProcessorIngestDocument, this.ingestDocument);
    assertThat(secondProcessorIngestDocument.getSourceAndMetadata(), not(sameInstance(this.ingestDocument.getSourceAndMetadata())));
    assertThat(secondProcessorIngestDocument.getSourceAndMetadata(), not(sameInstance(firstProcessorIngestDocument.getSourceAndMetadata())));
    assertThat(simulateDocumentVerboseResult.getProcessorResults().get(1).getFailure(), nullValue());
}

Example 82 with IngestDocument

use of org.elasticsearch.ingest.IngestDocument in project elasticsearch by elastic.

the class SimulateExecutionServiceTests method testExecuteVerboseItemWithOnFailure.

public void testExecuteVerboseItemWithOnFailure() throws Exception {
    TestProcessor processor1 = new TestProcessor("processor_0", "mock", ingestDocument -> {
        throw new RuntimeException("processor failed");
    });
    TestProcessor processor2 = new TestProcessor("processor_1", "mock", ingestDocument -> {
    });
    TestProcessor processor3 = new TestProcessor("processor_2", "mock", ingestDocument -> {
    });
    Pipeline pipeline = new Pipeline("_id", "_description", version, new CompoundProcessor(new CompoundProcessor(false, Collections.singletonList(processor1), Collections.singletonList(processor2)), processor3));
    SimulateDocumentResult actualItemResponse = executionService.executeDocument(pipeline, ingestDocument, true);
    assertThat(processor1.getInvokedCounter(), equalTo(1));
    assertThat(processor2.getInvokedCounter(), equalTo(1));
    assertThat(actualItemResponse, instanceOf(SimulateDocumentVerboseResult.class));
    SimulateDocumentVerboseResult simulateDocumentVerboseResult = (SimulateDocumentVerboseResult) actualItemResponse;
    assertThat(simulateDocumentVerboseResult.getProcessorResults().size(), equalTo(3));
    assertThat(simulateDocumentVerboseResult.getProcessorResults().get(0).getProcessorTag(), equalTo("processor_0"));
    assertThat(simulateDocumentVerboseResult.getProcessorResults().get(0).getIngestDocument(), nullValue());
    assertThat(simulateDocumentVerboseResult.getProcessorResults().get(0).getFailure(), instanceOf(RuntimeException.class));
    RuntimeException runtimeException = (RuntimeException) simulateDocumentVerboseResult.getProcessorResults().get(0).getFailure();
    assertThat(runtimeException.getMessage(), equalTo("processor failed"));
    assertThat(simulateDocumentVerboseResult.getProcessorResults().get(1).getProcessorTag(), equalTo("processor_1"));
    assertThat(simulateDocumentVerboseResult.getProcessorResults().get(1).getIngestDocument(), not(sameInstance(ingestDocument)));
    IngestDocument ingestDocumentWithOnFailureMetadata = new IngestDocument(ingestDocument);
    Map<String, Object> metadata = ingestDocumentWithOnFailureMetadata.getIngestMetadata();
    metadata.put(CompoundProcessor.ON_FAILURE_PROCESSOR_TYPE_FIELD, "mock");
    metadata.put(CompoundProcessor.ON_FAILURE_PROCESSOR_TAG_FIELD, "processor_0");
    metadata.put(CompoundProcessor.ON_FAILURE_MESSAGE_FIELD, "processor failed");
    assertIngestDocument(simulateDocumentVerboseResult.getProcessorResults().get(1).getIngestDocument(), ingestDocumentWithOnFailureMetadata);
    assertThat(simulateDocumentVerboseResult.getProcessorResults().get(1).getFailure(), nullValue());
    assertThat(simulateDocumentVerboseResult.getProcessorResults().get(2).getProcessorTag(), equalTo("processor_2"));
    assertThat(simulateDocumentVerboseResult.getProcessorResults().get(2).getIngestDocument(), not(sameInstance(ingestDocument)));
    assertIngestDocument(simulateDocumentVerboseResult.getProcessorResults().get(2).getIngestDocument(), ingestDocument);
    assertThat(simulateDocumentVerboseResult.getProcessorResults().get(2).getFailure(), nullValue());
}

Example 83 with IngestDocument

use of org.elasticsearch.ingest.IngestDocument in project elasticsearch by elastic.

the class SimulatePipelineRequestParsingTests method testParseUsingPipelineStore.

public void testParseUsingPipelineStore() throws Exception {
    int numDocs = randomIntBetween(1, 10);
    Map<String, Object> requestContent = new HashMap<>();
    List<Map<String, Object>> docs = new ArrayList<>();
    List<Map<String, Object>> expectedDocs = new ArrayList<>();
    requestContent.put(Fields.DOCS, docs);
    for (int i = 0; i < numDocs; i++) {
        Map<String, Object> doc = new HashMap<>();
        String index = randomAsciiOfLengthBetween(1, 10);
        String type = randomAsciiOfLengthBetween(1, 10);
        String id = randomAsciiOfLengthBetween(1, 10);
        doc.put(INDEX.getFieldName(), index);
        doc.put(TYPE.getFieldName(), type);
        doc.put(ID.getFieldName(), id);
        String fieldName = randomAsciiOfLengthBetween(1, 10);
        String fieldValue = randomAsciiOfLengthBetween(1, 10);
        doc.put(Fields.SOURCE, Collections.singletonMap(fieldName, fieldValue));
        docs.add(doc);
        Map<String, Object> expectedDoc = new HashMap<>();
        expectedDoc.put(INDEX.getFieldName(), index);
        expectedDoc.put(TYPE.getFieldName(), type);
        expectedDoc.put(ID.getFieldName(), id);
        expectedDoc.put(Fields.SOURCE, Collections.singletonMap(fieldName, fieldValue));
        expectedDocs.add(expectedDoc);
    }
    SimulatePipelineRequest.Parsed actualRequest = SimulatePipelineRequest.parseWithPipelineId(SIMULATED_PIPELINE_ID, requestContent, false, store);
    assertThat(actualRequest.isVerbose(), equalTo(false));
    assertThat(actualRequest.getDocuments().size(), equalTo(numDocs));
    Iterator<Map<String, Object>> expectedDocsIterator = expectedDocs.iterator();
    for (IngestDocument ingestDocument : actualRequest.getDocuments()) {
        Map<String, Object> expectedDocument = expectedDocsIterator.next();
        Map<IngestDocument.MetaData, String> metadataMap = ingestDocument.extractMetadata();
        assertThat(metadataMap.get(INDEX), equalTo(expectedDocument.get(INDEX.getFieldName())));
        assertThat(metadataMap.get(TYPE), equalTo(expectedDocument.get(TYPE.getFieldName())));
        assertThat(metadataMap.get(ID), equalTo(expectedDocument.get(ID.getFieldName())));
        assertThat(ingestDocument.getSourceAndMetadata(), equalTo(expectedDocument.get(Fields.SOURCE)));
    }
    assertThat(actualRequest.getPipeline().getId(), equalTo(SIMULATED_PIPELINE_ID));
    assertThat(actualRequest.getPipeline().getDescription(), nullValue());
    assertThat(actualRequest.getPipeline().getProcessors().size(), equalTo(1));
}

Example 84 with IngestDocument

use of org.elasticsearch.ingest.IngestDocument in project elasticsearch by elastic.

the class TrackingResultProcessorTests method init.

@Before
public void init() {
    ingestDocument = new IngestDocument(new HashMap<>(), new HashMap<>());
    resultList = new ArrayList<>();
}

Example 85 with IngestDocument

use of org.elasticsearch.ingest.IngestDocument in project elasticsearch by elastic.

the class KeyValueProcessorTests method testNullValueWithIgnoreMissing.

public void testNullValueWithIgnoreMissing() throws Exception {
    String fieldName = RandomDocumentPicks.randomFieldName(random());
    IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap(fieldName, null));
    IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
    Processor processor = new KeyValueProcessor(randomAsciiOfLength(10), fieldName, "", "", null, "target", true);
    processor.execute(ingestDocument);
    assertIngestDocument(originalIngestDocument, ingestDocument);
}