Search in sources :

Example 1 with Processor

use of org.opensearch.ingest.Processor in project OpenSearch by opensearch-project.

the class DotExpanderProcessorTests method testEscapeFields_valueField.

public void testEscapeFields_valueField() throws Exception {
    Map<String, Object> source = new HashMap<>();
    source.put("foo.bar", "baz1");
    source.put("foo", "baz2");
    IngestDocument document1 = new IngestDocument(source, Collections.emptyMap());
    Processor processor1 = new DotExpanderProcessor("_tag", null, null, "foo.bar");
    // foo already exists and if a leaf field and therefor can't be replaced by a map field:
    Exception e = expectThrows(IllegalArgumentException.class, () -> processor1.execute(document1));
    assertThat(e.getMessage(), equalTo("cannot expend [foo.bar], because [foo] is not an object field, but a value field"));
    // so because foo is no branch field but a value field the `foo.bar` field can't be expanded
    // into [foo].[bar], so foo should be renamed first into `[foo].[bar]:
    IngestDocument document = new IngestDocument(source, Collections.emptyMap());
    Processor processor = new RenameProcessor("_tag", null, new TestTemplateService.MockTemplateScript.Factory("foo"), new TestTemplateService.MockTemplateScript.Factory("foo.bar"), false);
    processor.execute(document);
    processor = new DotExpanderProcessor("_tag", null, null, "foo.bar");
    processor.execute(document);
    assertThat(document.getFieldValue("foo", Map.class).size(), equalTo(1));
    assertThat(document.getFieldValue("foo.bar.0", String.class), equalTo("baz2"));
    assertThat(document.getFieldValue("foo.bar.1", String.class), equalTo("baz1"));
    source = new HashMap<>();
    source.put("foo.bar", "baz1");
    document = new IngestDocument(source, Collections.emptyMap());
    processor = new DotExpanderProcessor("_tag", null, null, "foo.bar");
    processor.execute(document);
    assertThat(document.getFieldValue("foo", Map.class).size(), equalTo(1));
    assertThat(document.getFieldValue("foo.bar", String.class), equalTo("baz1"));
    source = new HashMap<>();
    source.put("foo.bar.baz", "baz1");
    source.put("foo", new HashMap<>(Collections.singletonMap("bar", new HashMap<>())));
    document = new IngestDocument(source, Collections.emptyMap());
    processor = new DotExpanderProcessor("_tag", null, null, "foo.bar.baz");
    processor.execute(document);
    assertThat(document.getFieldValue("foo", Map.class).size(), equalTo(1));
    assertThat(document.getFieldValue("foo.bar", Map.class).size(), equalTo(1));
    assertThat(document.getFieldValue("foo.bar.baz", String.class), equalTo("baz1"));
    source = new HashMap<>();
    source.put("foo.bar.baz", "baz1");
    source.put("foo", new HashMap<>(Collections.singletonMap("bar", "baz2")));
    IngestDocument document2 = new IngestDocument(source, Collections.emptyMap());
    Processor processor2 = new DotExpanderProcessor("_tag", null, null, "foo.bar.baz");
    e = expectThrows(IllegalArgumentException.class, () -> processor2.execute(document2));
    assertThat(e.getMessage(), equalTo("cannot expend [foo.bar.baz], because [foo.bar] is not an object field, but a value field"));
}
Also used : Processor(org.opensearch.ingest.Processor) HashMap(java.util.HashMap) IngestDocument(org.opensearch.ingest.IngestDocument)

Example 2 with Processor

use of org.opensearch.ingest.Processor in project OpenSearch by opensearch-project.

the class FailProcessorTests method test.

public void test() throws Exception {
    IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
    String message = randomAlphaOfLength(10);
    Processor processor = new FailProcessor(randomAlphaOfLength(10), null, new TestTemplateService.MockTemplateScript.Factory(message));
    try {
        processor.execute(ingestDocument);
        fail("fail processor should throw an exception");
    } catch (FailProcessorException e) {
        assertThat(e.getMessage(), equalTo(message));
    }
}
Also used : Processor(org.opensearch.ingest.Processor) IngestDocument(org.opensearch.ingest.IngestDocument)

Example 3 with Processor

use of org.opensearch.ingest.Processor in project OpenSearch by opensearch-project.

the class ForEachProcessorFactoryTests method testCreateWithTooManyProcessorTypes.

public void testCreateWithTooManyProcessorTypes() throws Exception {
    Processor processor = new TestProcessor(ingestDocument -> {
    });
    Map<String, Processor.Factory> registry = new HashMap<>();
    registry.put("_first", (r, t, description, c) -> processor);
    registry.put("_second", (r, t, description, c) -> processor);
    ForEachProcessor.Factory forEachFactory = new ForEachProcessor.Factory(scriptService);
    Map<String, Object> config = new HashMap<>();
    config.put("field", "_field");
    Map<String, Object> processorTypes = new HashMap<>();
    processorTypes.put("_first", Collections.emptyMap());
    processorTypes.put("_second", Collections.emptyMap());
    config.put("processor", processorTypes);
    Exception exception = expectThrows(OpenSearchParseException.class, () -> forEachFactory.create(registry, null, null, config));
    assertThat(exception.getMessage(), equalTo("[processor] Must specify exactly one processor type"));
}
Also used : TestProcessor(org.opensearch.ingest.TestProcessor) Processor(org.opensearch.ingest.Processor) HashMap(java.util.HashMap) TestProcessor(org.opensearch.ingest.TestProcessor) OpenSearchParseException(org.opensearch.OpenSearchParseException)

Example 4 with Processor

use of org.opensearch.ingest.Processor in project OpenSearch by opensearch-project.

the class ForEachProcessorFactoryTests method testSetIgnoreMissing.

public void testSetIgnoreMissing() throws Exception {
    Processor processor = new TestProcessor(ingestDocument -> {
    });
    Map<String, Processor.Factory> registry = new HashMap<>();
    registry.put("_name", (r, t, description, c) -> processor);
    ForEachProcessor.Factory forEachFactory = new ForEachProcessor.Factory(scriptService);
    Map<String, Object> config = new HashMap<>();
    config.put("field", "_field");
    config.put("processor", Collections.singletonMap("_name", Collections.emptyMap()));
    config.put("ignore_missing", true);
    ForEachProcessor forEachProcessor = forEachFactory.create(registry, null, null, config);
    assertThat(forEachProcessor, Matchers.notNullValue());
    assertThat(forEachProcessor.getField(), equalTo("_field"));
    assertThat(forEachProcessor.getInnerProcessor(), Matchers.sameInstance(processor));
    assertTrue(forEachProcessor.isIgnoreMissing());
}
Also used : TestProcessor(org.opensearch.ingest.TestProcessor) Processor(org.opensearch.ingest.Processor) HashMap(java.util.HashMap) TestProcessor(org.opensearch.ingest.TestProcessor)

Example 5 with Processor

use of org.opensearch.ingest.Processor in project OpenSearch by opensearch-project.

the class ForEachProcessorFactoryTests method testCreateWithMissingField.

public void testCreateWithMissingField() throws Exception {
    Processor processor = new TestProcessor(ingestDocument -> {
    });
    Map<String, Processor.Factory> registry = new HashMap<>();
    registry.put("_name", (r, t, description, c) -> processor);
    ForEachProcessor.Factory forEachFactory = new ForEachProcessor.Factory(scriptService);
    Map<String, Object> config = new HashMap<>();
    config.put("processor", Collections.singletonList(Collections.singletonMap("_name", Collections.emptyMap())));
    Exception exception = expectThrows(Exception.class, () -> forEachFactory.create(registry, null, null, config));
    assertThat(exception.getMessage(), equalTo("[field] required property is missing"));
}
Also used : TestProcessor(org.opensearch.ingest.TestProcessor) Processor(org.opensearch.ingest.Processor) HashMap(java.util.HashMap) TestProcessor(org.opensearch.ingest.TestProcessor) OpenSearchParseException(org.opensearch.OpenSearchParseException)

Aggregations

Processor (org.opensearch.ingest.Processor)143 IngestDocument (org.opensearch.ingest.IngestDocument)136 IngestDocumentMatcher.assertIngestDocument (org.opensearch.ingest.IngestDocumentMatcher.assertIngestDocument)88 Matchers.containsString (org.hamcrest.Matchers.containsString)67 ArrayList (java.util.ArrayList)43 List (java.util.List)34 HashMap (java.util.HashMap)21 TestProcessor (org.opensearch.ingest.TestProcessor)12 SortOrder (org.opensearch.ingest.common.SortProcessor.SortOrder)12 Map (java.util.Map)8 CompoundProcessor (org.opensearch.ingest.CompoundProcessor)8 Collections (java.util.Collections)5 BiConsumer (java.util.function.BiConsumer)5 Matchers.equalTo (org.hamcrest.Matchers.equalTo)5 TestTemplateService (org.opensearch.ingest.TestTemplateService)5 OpenSearchTestCase (org.opensearch.test.OpenSearchTestCase)5 Arrays (java.util.Arrays)4 Locale (java.util.Locale)4 CountDownLatch (java.util.concurrent.CountDownLatch)4 AtomicReference (java.util.concurrent.atomic.AtomicReference)4