Examples with IngestDocument org.opensearch.ingest.IngestDocument used on opensource projects

Example 76 with IngestDocument

use of org.opensearch.ingest.IngestDocument in project OpenSearch by opensearch-project.

the class ConvertProcessorTests method testConvertNonExistingField.

public void testConvertNonExistingField() throws Exception {
    IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
    String fieldName = RandomDocumentPicks.randomFieldName(random());
    Type type = randomFrom(Type.values());
    Processor processor = new ConvertProcessor(randomAlphaOfLength(10), null, fieldName, fieldName, type, false);
    try {
        processor.execute(ingestDocument);
        fail("processor execute should have failed");
    } catch (IllegalArgumentException e) {
        assertThat(e.getMessage(), containsString("not present as part of path [" + fieldName + "]"));
    }
}

Example 77 with IngestDocument

use of org.opensearch.ingest.IngestDocument in project OpenSearch by opensearch-project.

the class GeoIpProcessorFactoryTests method testLoadingCustomDatabase.

public void testLoadingCustomDatabase() throws IOException {
    final Path geoIpDir = createTempDir();
    final Path configDir = createTempDir();
    final Path geoIpConfigDir = configDir.resolve("ingest-geoip");
    Files.createDirectories(geoIpConfigDir);
    copyDatabaseFiles(geoIpDir);
    // fake the GeoIP2-City database
    copyDatabaseFile(geoIpConfigDir, "GeoLite2-City.mmdb");
    Files.move(geoIpConfigDir.resolve("GeoLite2-City.mmdb"), geoIpConfigDir.resolve("GeoIP2-City.mmdb"));
    /*
         * Loading another database reader instances, because otherwise we can't test lazy loading as the database readers used at class
         * level are reused between tests. (we want to keep that otherwise running this test will take roughly 4 times more time).
         */
    final Map<String, DatabaseReaderLazyLoader> databaseReaders = IngestGeoIpPlugin.loadDatabaseReaders(geoIpDir, geoIpConfigDir);
    final GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders, new GeoIpCache(1000));
    for (DatabaseReaderLazyLoader lazyLoader : databaseReaders.values()) {
        assertNull(lazyLoader.databaseReader.get());
    }
    final Map<String, Object> field = Collections.singletonMap("_field", "1.1.1.1");
    final IngestDocument document = new IngestDocument("index", "id", "routing", 1L, VersionType.EXTERNAL, field);
    Map<String, Object> config = new HashMap<>();
    config.put("field", "_field");
    config.put("database_file", "GeoIP2-City.mmdb");
    final GeoIpProcessor city = factory.create(null, "_tag", null, config);
    // these are lazy loaded until first use so we expect null here
    assertNull(databaseReaders.get("GeoIP2-City.mmdb").databaseReader.get());
    city.execute(document);
    // the first ingest should trigger a database load
    assertNotNull(databaseReaders.get("GeoIP2-City.mmdb").databaseReader.get());
}

Example 78 with IngestDocument

use of org.opensearch.ingest.IngestDocument in project OpenSearch by opensearch-project.

the class GeoIpProcessorFactoryTests method testLazyLoading.

public void testLazyLoading() throws Exception {
    final Path geoIpDir = createTempDir();
    final Path configDir = createTempDir();
    final Path geoIpConfigDir = configDir.resolve("ingest-geoip");
    Files.createDirectories(geoIpConfigDir);
    copyDatabaseFiles(geoIpDir);
    // Loading another database reader instances, because otherwise we can't test lazy loading as the
    // database readers used at class level are reused between tests. (we want to keep that otherwise running this
    // test will take roughly 4 times more time)
    Map<String, DatabaseReaderLazyLoader> databaseReaders = IngestGeoIpPlugin.loadDatabaseReaders(geoIpDir, geoIpConfigDir);
    GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders, new GeoIpCache(1000));
    for (DatabaseReaderLazyLoader lazyLoader : databaseReaders.values()) {
        assertNull(lazyLoader.databaseReader.get());
    }
    final Map<String, Object> field = Collections.singletonMap("_field", "1.1.1.1");
    final IngestDocument document = new IngestDocument("index", "id", "routing", 1L, VersionType.EXTERNAL, field);
    Map<String, Object> config = new HashMap<>();
    config.put("field", "_field");
    config.put("database_file", "GeoLite2-City.mmdb");
    final GeoIpProcessor city = factory.create(null, "_tag", null, config);
    // these are lazy loaded until first use so we expect null here
    assertNull(databaseReaders.get("GeoLite2-City.mmdb").databaseReader.get());
    city.execute(document);
    // the first ingest should trigger a database load
    assertNotNull(databaseReaders.get("GeoLite2-City.mmdb").databaseReader.get());
    config = new HashMap<>();
    config.put("field", "_field");
    config.put("database_file", "GeoLite2-Country.mmdb");
    final GeoIpProcessor country = factory.create(null, "_tag", null, config);
    // these are lazy loaded until first use so we expect null here
    assertNull(databaseReaders.get("GeoLite2-Country.mmdb").databaseReader.get());
    country.execute(document);
    // the first ingest should trigger a database load
    assertNotNull(databaseReaders.get("GeoLite2-Country.mmdb").databaseReader.get());
    config = new HashMap<>();
    config.put("field", "_field");
    config.put("database_file", "GeoLite2-ASN.mmdb");
    final GeoIpProcessor asn = factory.create(null, "_tag", null, config);
    // these are lazy loaded until first use so we expect null here
    assertNull(databaseReaders.get("GeoLite2-ASN.mmdb").databaseReader.get());
    asn.execute(document);
    // the first ingest should trigger a database load
    assertNotNull(databaseReaders.get("GeoLite2-ASN.mmdb").databaseReader.get());
}

Example 79 with IngestDocument

use of org.opensearch.ingest.IngestDocument in project OpenSearch by opensearch-project.

the class UserAgentProcessorTests method testUnknown.

@SuppressWarnings("unchecked")
public void testUnknown() throws Exception {
    Map<String, Object> document = new HashMap<>();
    document.put("source_field", "Something I made up v42.0.1");
    IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
    processor.execute(ingestDocument);
    Map<String, Object> data = ingestDocument.getSourceAndMetadata();
    assertThat(data, hasKey("target_field"));
    Map<String, Object> target = (Map<String, Object>) data.get("target_field");
    assertThat(target.get("name"), is("Other"));
    assertNull(target.get("major"));
    assertNull(target.get("minor"));
    assertNull(target.get("patch"));
    assertNull(target.get("build"));
    assertNull(target.get("os"));
    Map<String, String> device = new HashMap<>();
    device.put("name", "Other");
    assertThat(target.get("device"), is(device));
}

Example 80 with IngestDocument

use of org.opensearch.ingest.IngestDocument in project OpenSearch by opensearch-project.

the class UserAgentProcessorTests method testNullWithoutIgnoreMissing.

public void testNullWithoutIgnoreMissing() throws Exception {
    UserAgentProcessor processor = new UserAgentProcessor(randomAlphaOfLength(10), null, "source_field", "target_field", null, EnumSet.allOf(UserAgentProcessor.Property.class), false, true);
    IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), Collections.singletonMap("source_field", null));
    IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
    Exception exception = expectThrows(Exception.class, () -> processor.execute(ingestDocument));
    assertThat(exception.getMessage(), equalTo("field [source_field] is null, cannot parse user-agent."));
}