Examples with IngestDocument org.opensearch.ingest.IngestDocument used on opensource projects

Example 56 with IngestDocument

use of org.opensearch.ingest.IngestDocument in project OpenSearch by opensearch-project.

the class CsvProcessorTests method processDocument.

private IngestDocument processDocument(String[] headers, String csv, boolean trim, Object emptyValue) {
    IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
    Arrays.stream(headers).filter(ingestDocument::hasField).forEach(ingestDocument::removeField);
    String fieldName = randomAlphaOfLength(11);
    ingestDocument.setFieldValue(fieldName, csv);
    char quoteChar = quote.isEmpty() ? '"' : quote.charAt(0);
    CsvProcessor processor = new CsvProcessor(randomAlphaOfLength(5), null, fieldName, headers, trim, separator, quoteChar, false, emptyValue);
    processor.execute(ingestDocument);
    return ingestDocument;
}

Example 57 with IngestDocument

use of org.opensearch.ingest.IngestDocument in project OpenSearch by opensearch-project.

the class CsvProcessorTests method testEscapedQuote.

public void testEscapedQuote() {
    int numItems = randomIntBetween(2, 10);
    Map<String, String> items = new LinkedHashMap<>();
    for (int i = 0; i < numItems; i++) {
        items.put(randomAlphaOfLengthBetween(5, 10), randomAlphaOfLengthBetween(5, 10) + quote + quote + randomAlphaOfLengthBetween(5, 10) + quote + quote);
    }
    String[] headers = items.keySet().toArray(new String[numItems]);
    String csv = items.values().stream().map(v -> quote + v + quote).collect(Collectors.joining(separator + ""));
    IngestDocument ingestDocument = processDocument(headers, csv);
    items.forEach((key, value) -> assertEquals(value.replace(quote + quote, quote), ingestDocument.getFieldValue(key, String.class)));
}

Example 58 with IngestDocument

use of org.opensearch.ingest.IngestDocument in project OpenSearch by opensearch-project.

the class CsvProcessorTests method testExactNumberOfFields.

public void testExactNumberOfFields() {
    int numItems = randomIntBetween(2, 10);
    Map<String, String> items = new LinkedHashMap<>();
    for (int i = 0; i < numItems; i++) {
        items.put(randomAlphaOfLengthBetween(5, 10), randomAlphaOfLengthBetween(5, 10));
    }
    String[] headers = items.keySet().toArray(new String[numItems]);
    String csv = items.values().stream().map(v -> quote + v + quote).collect(Collectors.joining(separator + ""));
    IngestDocument ingestDocument = processDocument(headers, csv);
    items.forEach((key, value) -> assertEquals(value, ingestDocument.getFieldValue(key, String.class)));
}

Example 59 with IngestDocument

use of org.opensearch.ingest.IngestDocument in project OpenSearch by opensearch-project.

the class GrokProcessorTests method testMultiplePatternsWithMatchReturn.

public void testMultiplePatternsWithMatchReturn() throws Exception {
    String fieldName = RandomDocumentPicks.randomFieldName(random());
    IngestDocument doc = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
    doc.setFieldValue(fieldName, "2");
    Map<String, String> patternBank = new HashMap<>();
    patternBank.put("ONE", "1");
    patternBank.put("TWO", "2");
    patternBank.put("THREE", "3");
    GrokProcessor processor = new GrokProcessor(randomAlphaOfLength(10), null, patternBank, Arrays.asList("%{ONE:one}", "%{TWO:two}", "%{THREE:three}"), fieldName, false, false, MatcherWatchdog.noop());
    processor.execute(doc);
    assertThat(doc.hasField("one"), equalTo(false));
    assertThat(doc.getFieldValue("two", String.class), equalTo("2"));
    assertThat(doc.hasField("three"), equalTo(false));
}

Example 60 with IngestDocument

use of org.opensearch.ingest.IngestDocument in project OpenSearch by opensearch-project.

the class GrokProcessorTests method testIgnoreCase.

public void testIgnoreCase() throws Exception {
    String fieldName = RandomDocumentPicks.randomFieldName(random());
    IngestDocument doc = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
    doc.setFieldValue(fieldName, "A");
    GrokProcessor processor = new GrokProcessor(randomAlphaOfLength(10), null, Collections.emptyMap(), Collections.singletonList("(?<a>(?i)A)"), fieldName, false, false, MatcherWatchdog.noop());
    processor.execute(doc);
    assertThat(doc.getFieldValue("a", String.class), equalTo("A"));
}