Examples with GeoIpCache org.opensearch.ingest.geoip.IngestGeoIpPlugin.GeoIpCache used on opensource projects

Example 1 with GeoIpCache

use of org.opensearch.ingest.geoip.IngestGeoIpPlugin.GeoIpCache in project OpenSearch by opensearch-project.

the class GeoIpProcessorFactoryTests method testLoadingCustomDatabase.

public void testLoadingCustomDatabase() throws IOException {
    final Path geoIpDir = createTempDir();
    final Path configDir = createTempDir();
    final Path geoIpConfigDir = configDir.resolve("ingest-geoip");
    Files.createDirectories(geoIpConfigDir);
    copyDatabaseFiles(geoIpDir);
    // fake the GeoIP2-City database
    copyDatabaseFile(geoIpConfigDir, "GeoLite2-City.mmdb");
    Files.move(geoIpConfigDir.resolve("GeoLite2-City.mmdb"), geoIpConfigDir.resolve("GeoIP2-City.mmdb"));
    /*
         * Loading another database reader instances, because otherwise we can't test lazy loading as the database readers used at class
         * level are reused between tests. (we want to keep that otherwise running this test will take roughly 4 times more time).
         */
    final Map<String, DatabaseReaderLazyLoader> databaseReaders = IngestGeoIpPlugin.loadDatabaseReaders(geoIpDir, geoIpConfigDir);
    final GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders, new GeoIpCache(1000));
    for (DatabaseReaderLazyLoader lazyLoader : databaseReaders.values()) {
        assertNull(lazyLoader.databaseReader.get());
    }
    final Map<String, Object> field = Collections.singletonMap("_field", "1.1.1.1");
    final IngestDocument document = new IngestDocument("index", "id", "routing", 1L, VersionType.EXTERNAL, field);
    Map<String, Object> config = new HashMap<>();
    config.put("field", "_field");
    config.put("database_file", "GeoIP2-City.mmdb");
    final GeoIpProcessor city = factory.create(null, "_tag", null, config);
    // these are lazy loaded until first use so we expect null here
    assertNull(databaseReaders.get("GeoIP2-City.mmdb").databaseReader.get());
    city.execute(document);
    // the first ingest should trigger a database load
    assertNotNull(databaseReaders.get("GeoIP2-City.mmdb").databaseReader.get());
}

Example 2 with GeoIpCache

use of org.opensearch.ingest.geoip.IngestGeoIpPlugin.GeoIpCache in project OpenSearch by opensearch-project.

the class GeoIpProcessorFactoryTests method testBuildWithCountryDbAndAsnFields.

public void testBuildWithCountryDbAndAsnFields() throws Exception {
    GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders, new GeoIpCache(1000));
    Map<String, Object> config = new HashMap<>();
    config.put("field", "_field");
    config.put("database_file", "GeoLite2-Country.mmdb");
    EnumSet<GeoIpProcessor.Property> asnOnlyProperties = EnumSet.copyOf(GeoIpProcessor.Property.ALL_ASN_PROPERTIES);
    asnOnlyProperties.remove(GeoIpProcessor.Property.IP);
    String asnProperty = RandomPicks.randomFrom(Randomness.get(), asnOnlyProperties).toString();
    config.put("properties", Collections.singletonList(asnProperty));
    Exception e = expectThrows(OpenSearchParseException.class, () -> factory.create(null, null, null, config));
    assertThat(e.getMessage(), equalTo("[properties] illegal property value [" + asnProperty + "]. valid values are [IP, COUNTRY_ISO_CODE, COUNTRY_NAME, CONTINENT_NAME]"));
}

Example 3 with GeoIpCache

use of org.opensearch.ingest.geoip.IngestGeoIpPlugin.GeoIpCache in project OpenSearch by opensearch-project.

the class GeoIpProcessorFactoryTests method testBuildFields.

public void testBuildFields() throws Exception {
    GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders, new GeoIpCache(1000));
    Set<GeoIpProcessor.Property> properties = EnumSet.noneOf(GeoIpProcessor.Property.class);
    List<String> fieldNames = new ArrayList<>();
    int counter = 0;
    int numFields = scaledRandomIntBetween(1, GeoIpProcessor.Property.values().length);
    for (GeoIpProcessor.Property property : GeoIpProcessor.Property.ALL_CITY_PROPERTIES) {
        properties.add(property);
        fieldNames.add(property.name().toLowerCase(Locale.ROOT));
        if (++counter >= numFields) {
            break;
        }
    }
    Map<String, Object> config = new HashMap<>();
    config.put("field", "_field");
    config.put("properties", fieldNames);
    GeoIpProcessor processor = factory.create(null, null, null, config);
    assertThat(processor.getField(), equalTo("_field"));
    assertThat(processor.getProperties(), equalTo(properties));
    assertFalse(processor.isIgnoreMissing());
}

Example 4 with GeoIpCache

use of org.opensearch.ingest.geoip.IngestGeoIpPlugin.GeoIpCache in project OpenSearch by opensearch-project.

the class GeoIpProcessorFactoryTests method testBuildTargetField.

public void testBuildTargetField() throws Exception {
    GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders, new GeoIpCache(1000));
    Map<String, Object> config = new HashMap<>();
    config.put("field", "_field");
    config.put("target_field", "_field");
    GeoIpProcessor processor = factory.create(null, null, null, config);
    assertThat(processor.getField(), equalTo("_field"));
    assertThat(processor.getTargetField(), equalTo("_field"));
    assertFalse(processor.isIgnoreMissing());
}

Example 5 with GeoIpCache

use of org.opensearch.ingest.geoip.IngestGeoIpPlugin.GeoIpCache in project OpenSearch by opensearch-project.

the class GeoIpProcessorFactoryTests method testLazyLoading.

public void testLazyLoading() throws Exception {
    final Path geoIpDir = createTempDir();
    final Path configDir = createTempDir();
    final Path geoIpConfigDir = configDir.resolve("ingest-geoip");
    Files.createDirectories(geoIpConfigDir);
    copyDatabaseFiles(geoIpDir);
    // Loading another database reader instances, because otherwise we can't test lazy loading as the
    // database readers used at class level are reused between tests. (we want to keep that otherwise running this
    // test will take roughly 4 times more time)
    Map<String, DatabaseReaderLazyLoader> databaseReaders = IngestGeoIpPlugin.loadDatabaseReaders(geoIpDir, geoIpConfigDir);
    GeoIpProcessor.Factory factory = new GeoIpProcessor.Factory(databaseReaders, new GeoIpCache(1000));
    for (DatabaseReaderLazyLoader lazyLoader : databaseReaders.values()) {
        assertNull(lazyLoader.databaseReader.get());
    }
    final Map<String, Object> field = Collections.singletonMap("_field", "1.1.1.1");
    final IngestDocument document = new IngestDocument("index", "id", "routing", 1L, VersionType.EXTERNAL, field);
    Map<String, Object> config = new HashMap<>();
    config.put("field", "_field");
    config.put("database_file", "GeoLite2-City.mmdb");
    final GeoIpProcessor city = factory.create(null, "_tag", null, config);
    // these are lazy loaded until first use so we expect null here
    assertNull(databaseReaders.get("GeoLite2-City.mmdb").databaseReader.get());
    city.execute(document);
    // the first ingest should trigger a database load
    assertNotNull(databaseReaders.get("GeoLite2-City.mmdb").databaseReader.get());
    config = new HashMap<>();
    config.put("field", "_field");
    config.put("database_file", "GeoLite2-Country.mmdb");
    final GeoIpProcessor country = factory.create(null, "_tag", null, config);
    // these are lazy loaded until first use so we expect null here
    assertNull(databaseReaders.get("GeoLite2-Country.mmdb").databaseReader.get());
    country.execute(document);
    // the first ingest should trigger a database load
    assertNotNull(databaseReaders.get("GeoLite2-Country.mmdb").databaseReader.get());
    config = new HashMap<>();
    config.put("field", "_field");
    config.put("database_file", "GeoLite2-ASN.mmdb");
    final GeoIpProcessor asn = factory.create(null, "_tag", null, config);
    // these are lazy loaded until first use so we expect null here
    assertNull(databaseReaders.get("GeoLite2-ASN.mmdb").databaseReader.get());
    asn.execute(document);
    // the first ingest should trigger a database load
    assertNotNull(databaseReaders.get("GeoLite2-ASN.mmdb").databaseReader.get());
}