Examples with IngestDocument org.elasticsearch.ingest.IngestDocument used on opensource projects

Example 1 with IngestDocument

use of org.elasticsearch.ingest.IngestDocument in project elasticsearch by elastic.

the class SimulatePipelineRequestParsingTests method testParseWithProvidedPipeline.

public void testParseWithProvidedPipeline() throws Exception {
    int numDocs = randomIntBetween(1, 10);
    Map<String, Object> requestContent = new HashMap<>();
    List<Map<String, Object>> docs = new ArrayList<>();
    List<Map<String, Object>> expectedDocs = new ArrayList<>();
    requestContent.put(Fields.DOCS, docs);
    for (int i = 0; i < numDocs; i++) {
        Map<String, Object> doc = new HashMap<>();
        String index = randomAsciiOfLengthBetween(1, 10);
        String type = randomAsciiOfLengthBetween(1, 10);
        String id = randomAsciiOfLengthBetween(1, 10);
        doc.put(INDEX.getFieldName(), index);
        doc.put(TYPE.getFieldName(), type);
        doc.put(ID.getFieldName(), id);
        String fieldName = randomAsciiOfLengthBetween(1, 10);
        String fieldValue = randomAsciiOfLengthBetween(1, 10);
        doc.put(Fields.SOURCE, Collections.singletonMap(fieldName, fieldValue));
        docs.add(doc);
        Map<String, Object> expectedDoc = new HashMap<>();
        expectedDoc.put(INDEX.getFieldName(), index);
        expectedDoc.put(TYPE.getFieldName(), type);
        expectedDoc.put(ID.getFieldName(), id);
        expectedDoc.put(Fields.SOURCE, Collections.singletonMap(fieldName, fieldValue));
        expectedDocs.add(expectedDoc);
    }
    Map<String, Object> pipelineConfig = new HashMap<>();
    List<Map<String, Object>> processors = new ArrayList<>();
    int numProcessors = randomIntBetween(1, 10);
    for (int i = 0; i < numProcessors; i++) {
        Map<String, Object> processorConfig = new HashMap<>();
        List<Map<String, Object>> onFailureProcessors = new ArrayList<>();
        int numOnFailureProcessors = randomIntBetween(0, 1);
        for (int j = 0; j < numOnFailureProcessors; j++) {
            onFailureProcessors.add(Collections.singletonMap("mock_processor", Collections.emptyMap()));
        }
        if (numOnFailureProcessors > 0) {
            processorConfig.put("on_failure", onFailureProcessors);
        }
        processors.add(Collections.singletonMap("mock_processor", processorConfig));
    }
    pipelineConfig.put("processors", processors);
    List<Map<String, Object>> onFailureProcessors = new ArrayList<>();
    int numOnFailureProcessors = randomIntBetween(0, 1);
    for (int i = 0; i < numOnFailureProcessors; i++) {
        onFailureProcessors.add(Collections.singletonMap("mock_processor", Collections.emptyMap()));
    }
    if (numOnFailureProcessors > 0) {
        pipelineConfig.put("on_failure", onFailureProcessors);
    }
    requestContent.put(Fields.PIPELINE, pipelineConfig);
    SimulatePipelineRequest.Parsed actualRequest = SimulatePipelineRequest.parse(requestContent, false, store);
    assertThat(actualRequest.isVerbose(), equalTo(false));
    assertThat(actualRequest.getDocuments().size(), equalTo(numDocs));
    Iterator<Map<String, Object>> expectedDocsIterator = expectedDocs.iterator();
    for (IngestDocument ingestDocument : actualRequest.getDocuments()) {
        Map<String, Object> expectedDocument = expectedDocsIterator.next();
        Map<IngestDocument.MetaData, String> metadataMap = ingestDocument.extractMetadata();
        assertThat(metadataMap.get(INDEX), equalTo(expectedDocument.get(INDEX.getFieldName())));
        assertThat(metadataMap.get(TYPE), equalTo(expectedDocument.get(TYPE.getFieldName())));
        assertThat(metadataMap.get(ID), equalTo(expectedDocument.get(ID.getFieldName())));
        assertThat(ingestDocument.getSourceAndMetadata(), equalTo(expectedDocument.get(Fields.SOURCE)));
    }
    assertThat(actualRequest.getPipeline().getId(), equalTo(SIMULATED_PIPELINE_ID));
    assertThat(actualRequest.getPipeline().getDescription(), nullValue());
    assertThat(actualRequest.getPipeline().getProcessors().size(), equalTo(numProcessors));
}

Example 2 with IngestDocument

use of org.elasticsearch.ingest.IngestDocument in project elasticsearch by elastic.

the class SimulatePipelineResponseTests method testSerialization.

public void testSerialization() throws IOException {
    boolean isVerbose = randomBoolean();
    String id = randomBoolean() ? randomAsciiOfLengthBetween(1, 10) : null;
    int numResults = randomIntBetween(1, 10);
    List<SimulateDocumentResult> results = new ArrayList<>(numResults);
    for (int i = 0; i < numResults; i++) {
        boolean isFailure = randomBoolean();
        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
        if (isVerbose) {
            int numProcessors = randomIntBetween(1, 10);
            List<SimulateProcessorResult> processorResults = new ArrayList<>(numProcessors);
            for (int j = 0; j < numProcessors; j++) {
                String processorTag = randomAsciiOfLengthBetween(1, 10);
                SimulateProcessorResult processorResult;
                if (isFailure) {
                    processorResult = new SimulateProcessorResult(processorTag, new IllegalArgumentException("test"));
                } else {
                    processorResult = new SimulateProcessorResult(processorTag, ingestDocument);
                }
                processorResults.add(processorResult);
            }
            results.add(new SimulateDocumentVerboseResult(processorResults));
        } else {
            results.add(new SimulateDocumentBaseResult(ingestDocument));
            SimulateDocumentBaseResult simulateDocumentBaseResult;
            if (isFailure) {
                simulateDocumentBaseResult = new SimulateDocumentBaseResult(new IllegalArgumentException("test"));
            } else {
                simulateDocumentBaseResult = new SimulateDocumentBaseResult(ingestDocument);
            }
            results.add(simulateDocumentBaseResult);
        }
    }
    SimulatePipelineResponse response = new SimulatePipelineResponse(id, isVerbose, results);
    BytesStreamOutput out = new BytesStreamOutput();
    response.writeTo(out);
    StreamInput streamInput = out.bytes().streamInput();
    SimulatePipelineResponse otherResponse = new SimulatePipelineResponse();
    otherResponse.readFrom(streamInput);
    assertThat(otherResponse.getPipelineId(), equalTo(response.getPipelineId()));
    assertThat(otherResponse.getResults().size(), equalTo(response.getResults().size()));
    Iterator<SimulateDocumentResult> expectedResultIterator = response.getResults().iterator();
    for (SimulateDocumentResult result : otherResponse.getResults()) {
        if (isVerbose) {
            SimulateDocumentVerboseResult expectedSimulateDocumentVerboseResult = (SimulateDocumentVerboseResult) expectedResultIterator.next();
            assertThat(result, instanceOf(SimulateDocumentVerboseResult.class));
            SimulateDocumentVerboseResult simulateDocumentVerboseResult = (SimulateDocumentVerboseResult) result;
            assertThat(simulateDocumentVerboseResult.getProcessorResults().size(), equalTo(expectedSimulateDocumentVerboseResult.getProcessorResults().size()));
            Iterator<SimulateProcessorResult> expectedProcessorResultIterator = expectedSimulateDocumentVerboseResult.getProcessorResults().iterator();
            for (SimulateProcessorResult simulateProcessorResult : simulateDocumentVerboseResult.getProcessorResults()) {
                SimulateProcessorResult expectedProcessorResult = expectedProcessorResultIterator.next();
                assertThat(simulateProcessorResult.getProcessorTag(), equalTo(expectedProcessorResult.getProcessorTag()));
                if (simulateProcessorResult.getIngestDocument() != null) {
                    assertIngestDocument(simulateProcessorResult.getIngestDocument(), expectedProcessorResult.getIngestDocument());
                }
                if (expectedProcessorResult.getFailure() == null) {
                    assertThat(simulateProcessorResult.getFailure(), nullValue());
                } else {
                    assertThat(simulateProcessorResult.getFailure(), instanceOf(IllegalArgumentException.class));
                    IllegalArgumentException e = (IllegalArgumentException) simulateProcessorResult.getFailure();
                    assertThat(e.getMessage(), equalTo("test"));
                }
            }
        } else {
            SimulateDocumentBaseResult expectedSimulateDocumentBaseResult = (SimulateDocumentBaseResult) expectedResultIterator.next();
            assertThat(result, instanceOf(SimulateDocumentBaseResult.class));
            SimulateDocumentBaseResult simulateDocumentBaseResult = (SimulateDocumentBaseResult) result;
            if (simulateDocumentBaseResult.getIngestDocument() != null) {
                assertIngestDocument(simulateDocumentBaseResult.getIngestDocument(), expectedSimulateDocumentBaseResult.getIngestDocument());
            }
            if (expectedSimulateDocumentBaseResult.getFailure() == null) {
                assertThat(simulateDocumentBaseResult.getFailure(), nullValue());
            } else {
                assertThat(simulateDocumentBaseResult.getFailure(), instanceOf(IllegalArgumentException.class));
                IllegalArgumentException e = (IllegalArgumentException) simulateDocumentBaseResult.getFailure();
                assertThat(e.getMessage(), equalTo("test"));
            }
        }
    }
}

Example 3 with IngestDocument

use of org.elasticsearch.ingest.IngestDocument in project elasticsearch by elastic.

the class SimulateProcessorResultTests method testSerialization.

public void testSerialization() throws IOException {
    String processorTag = randomAsciiOfLengthBetween(1, 10);
    boolean isSuccessful = randomBoolean();
    boolean isIgnoredException = randomBoolean();
    SimulateProcessorResult simulateProcessorResult;
    if (isSuccessful) {
        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
        if (isIgnoredException) {
            simulateProcessorResult = new SimulateProcessorResult(processorTag, ingestDocument, new IllegalArgumentException("test"));
        } else {
            simulateProcessorResult = new SimulateProcessorResult(processorTag, ingestDocument);
        }
    } else {
        simulateProcessorResult = new SimulateProcessorResult(processorTag, new IllegalArgumentException("test"));
    }
    BytesStreamOutput out = new BytesStreamOutput();
    simulateProcessorResult.writeTo(out);
    StreamInput streamInput = out.bytes().streamInput();
    SimulateProcessorResult otherSimulateProcessorResult = new SimulateProcessorResult(streamInput);
    assertThat(otherSimulateProcessorResult.getProcessorTag(), equalTo(simulateProcessorResult.getProcessorTag()));
    if (isSuccessful) {
        assertIngestDocument(otherSimulateProcessorResult.getIngestDocument(), simulateProcessorResult.getIngestDocument());
        if (isIgnoredException) {
            assertThat(otherSimulateProcessorResult.getFailure(), instanceOf(IllegalArgumentException.class));
            IllegalArgumentException e = (IllegalArgumentException) otherSimulateProcessorResult.getFailure();
            assertThat(e.getMessage(), equalTo("test"));
        } else {
            assertThat(otherSimulateProcessorResult.getFailure(), nullValue());
        }
    } else {
        assertThat(otherSimulateProcessorResult.getIngestDocument(), is(nullValue()));
        assertThat(otherSimulateProcessorResult.getFailure(), instanceOf(IllegalArgumentException.class));
        IllegalArgumentException e = (IllegalArgumentException) otherSimulateProcessorResult.getFailure();
        assertThat(e.getMessage(), equalTo("test"));
    }
}

Example 4 with IngestDocument

use of org.elasticsearch.ingest.IngestDocument in project elasticsearch by elastic.

the class WriteableIngestDocumentTests method testEqualsAndHashcode.

public void testEqualsAndHashcode() throws Exception {
    Map<String, Object> sourceAndMetadata = RandomDocumentPicks.randomSource(random());
    int numFields = randomIntBetween(1, IngestDocument.MetaData.values().length);
    for (int i = 0; i < numFields; i++) {
        sourceAndMetadata.put(randomFrom(IngestDocument.MetaData.values()).getFieldName(), randomAsciiOfLengthBetween(5, 10));
    }
    Map<String, Object> ingestMetadata = new HashMap<>();
    numFields = randomIntBetween(1, 5);
    for (int i = 0; i < numFields; i++) {
        ingestMetadata.put(randomAsciiOfLengthBetween(5, 10), randomAsciiOfLengthBetween(5, 10));
    }
    WriteableIngestDocument ingestDocument = new WriteableIngestDocument(new IngestDocument(sourceAndMetadata, ingestMetadata));
    boolean changed = false;
    Map<String, Object> otherSourceAndMetadata;
    if (randomBoolean()) {
        otherSourceAndMetadata = RandomDocumentPicks.randomSource(random());
        changed = true;
    } else {
        otherSourceAndMetadata = new HashMap<>(sourceAndMetadata);
    }
    if (randomBoolean()) {
        numFields = randomIntBetween(1, IngestDocument.MetaData.values().length);
        for (int i = 0; i < numFields; i++) {
            otherSourceAndMetadata.put(randomFrom(IngestDocument.MetaData.values()).getFieldName(), randomAsciiOfLengthBetween(5, 10));
        }
        changed = true;
    }
    Map<String, Object> otherIngestMetadata;
    if (randomBoolean()) {
        otherIngestMetadata = new HashMap<>();
        numFields = randomIntBetween(1, 5);
        for (int i = 0; i < numFields; i++) {
            otherIngestMetadata.put(randomAsciiOfLengthBetween(5, 10), randomAsciiOfLengthBetween(5, 10));
        }
        changed = true;
    } else {
        otherIngestMetadata = Collections.unmodifiableMap(ingestMetadata);
    }
    WriteableIngestDocument otherIngestDocument = new WriteableIngestDocument(new IngestDocument(otherSourceAndMetadata, otherIngestMetadata));
    if (changed) {
        assertThat(ingestDocument, not(equalTo(otherIngestDocument)));
        assertThat(otherIngestDocument, not(equalTo(ingestDocument)));
    } else {
        assertThat(ingestDocument, equalTo(otherIngestDocument));
        assertThat(otherIngestDocument, equalTo(ingestDocument));
        assertThat(ingestDocument.hashCode(), equalTo(otherIngestDocument.hashCode()));
        WriteableIngestDocument thirdIngestDocument = new WriteableIngestDocument(new IngestDocument(Collections.unmodifiableMap(sourceAndMetadata), Collections.unmodifiableMap(ingestMetadata)));
        assertThat(thirdIngestDocument, equalTo(ingestDocument));
        assertThat(ingestDocument, equalTo(thirdIngestDocument));
        assertThat(ingestDocument.hashCode(), equalTo(thirdIngestDocument.hashCode()));
    }
}

Example 5 with IngestDocument

use of org.elasticsearch.ingest.IngestDocument in project elasticsearch by elastic.

the class WriteableIngestDocumentTests method testSerialization.

public void testSerialization() throws IOException {
    Map<String, Object> sourceAndMetadata = RandomDocumentPicks.randomSource(random());
    int numFields = randomIntBetween(1, IngestDocument.MetaData.values().length);
    for (int i = 0; i < numFields; i++) {
        sourceAndMetadata.put(randomFrom(IngestDocument.MetaData.values()).getFieldName(), randomAsciiOfLengthBetween(5, 10));
    }
    Map<String, Object> ingestMetadata = new HashMap<>();
    numFields = randomIntBetween(1, 5);
    for (int i = 0; i < numFields; i++) {
        ingestMetadata.put(randomAsciiOfLengthBetween(5, 10), randomAsciiOfLengthBetween(5, 10));
    }
    WriteableIngestDocument writeableIngestDocument = new WriteableIngestDocument(new IngestDocument(sourceAndMetadata, ingestMetadata));
    BytesStreamOutput out = new BytesStreamOutput();
    writeableIngestDocument.writeTo(out);
    StreamInput streamInput = out.bytes().streamInput();
    WriteableIngestDocument otherWriteableIngestDocument = new WriteableIngestDocument(streamInput);
    assertIngestDocument(otherWriteableIngestDocument.getIngestDocument(), writeableIngestDocument.getIngestDocument());
}