Examples with ElasticsearchDao org.apache.metron.elasticsearch.dao.ElasticsearchDao used on opensource projects

Example 1 with ElasticsearchDao

use of org.apache.metron.elasticsearch.dao.ElasticsearchDao in project metron by apache.

the class ElasticsearchMetaAlertIntegrationTest method setupBefore.

@BeforeClass
public static void setupBefore() throws Exception {
    // setup the client
    es = new ElasticSearchComponent.Builder().withHttpPort(9211).withIndexDir(new File(INDEX_DIR)).build();
    es.start();
    AccessConfig accessConfig = new AccessConfig();
    Map<String, Object> globalConfig = new HashMap<String, Object>() {

        {
            put("es.clustername", "metron");
            put("es.port", "9300");
            put("es.ip", "localhost");
            put("es.date.format", DATE_FORMAT);
        }
    };
    accessConfig.setMaxSearchResults(1000);
    accessConfig.setGlobalConfigSupplier(() -> globalConfig);
    accessConfig.setMaxSearchGroups(100);
    esDao = new ElasticsearchDao();
    esDao.init(accessConfig);
    metaDao = new ElasticsearchMetaAlertDao(esDao);
}

Example 2 with ElasticsearchDao

use of org.apache.metron.elasticsearch.dao.ElasticsearchDao in project metron by apache.

the class ElasticsearchMetaAlertIntegrationTest method shouldSearchByNestedAlert.

@Test
public void shouldSearchByNestedAlert() throws Exception {
    // Load alerts
    List<Map<String, Object>> alerts = buildAlerts(4);
    alerts.get(0).put(METAALERT_FIELD, Collections.singletonList("meta_active"));
    alerts.get(0).put("ip_src_addr", "192.168.1.1");
    alerts.get(0).put("ip_src_port", 8010);
    alerts.get(1).put(METAALERT_FIELD, Collections.singletonList("meta_active"));
    alerts.get(1).put("ip_src_addr", "192.168.1.2");
    alerts.get(1).put("ip_src_port", 8009);
    alerts.get(2).put("ip_src_addr", "192.168.1.3");
    alerts.get(2).put("ip_src_port", 8008);
    alerts.get(3).put("ip_src_addr", "192.168.1.4");
    alerts.get(3).put("ip_src_port", 8007);
    elasticsearchAdd(alerts, INDEX, SENSOR_NAME);
    // Put the nested type into the test index, so that it'll match appropriately
    ((ElasticsearchDao) esDao).getClient().admin().indices().preparePutMapping(INDEX).setType("test_doc").setSource(nestedAlertMapping).get();
    // Load metaAlerts
    Map<String, Object> activeMetaAlert = buildMetaAlert("meta_active", MetaAlertStatus.ACTIVE, Optional.of(Arrays.asList(alerts.get(0), alerts.get(1))));
    Map<String, Object> inactiveMetaAlert = buildMetaAlert("meta_inactive", MetaAlertStatus.INACTIVE, Optional.of(Arrays.asList(alerts.get(2), alerts.get(3))));
    // We pass MetaAlertDao.METAALERT_TYPE, because the "_doc" gets appended automatically.
    elasticsearchAdd(Arrays.asList(activeMetaAlert, inactiveMetaAlert), METAALERTS_INDEX, MetaAlertDao.METAALERT_TYPE);
    // Verify load was successful
    findCreatedDocs(Arrays.asList(new GetRequest("message_0", SENSOR_NAME), new GetRequest("message_1", SENSOR_NAME), new GetRequest("message_2", SENSOR_NAME), new GetRequest("message_3", SENSOR_NAME), new GetRequest("meta_active", METAALERT_TYPE), new GetRequest("meta_inactive", METAALERT_TYPE)));
    SearchResponse searchResponse = metaDao.search(new SearchRequest() {

        {
            setQuery("(ip_src_addr:192.168.1.1 AND ip_src_port:8009) OR (alert.ip_src_addr:192.168.1.1 AND alert.ip_src_port:8009)");
            setIndices(Collections.singletonList(MetaAlertDao.METAALERT_TYPE));
            setFrom(0);
            setSize(5);
            setSort(Collections.singletonList(new SortField() {

                {
                    setField(Constants.GUID);
                }
            }));
        }
    });
    // Should not have results because nested alerts shouldn't be flattened
    Assert.assertEquals(0, searchResponse.getTotal());
    // Query against all indices. Only the single active meta alert should be returned.
    // The child alerts should be hidden.
    searchResponse = metaDao.search(new SearchRequest() {

        {
            setQuery("(ip_src_addr:192.168.1.1 AND ip_src_port:8010)" + " OR (alert.ip_src_addr:192.168.1.1 AND alert.ip_src_port:8010)");
            setIndices(Collections.singletonList("*"));
            setFrom(0);
            setSize(5);
            setSort(Collections.singletonList(new SortField() {

                {
                    setField(Constants.GUID);
                }
            }));
        }
    });
    // Nested query should match a nested alert
    Assert.assertEquals(1, searchResponse.getTotal());
    Assert.assertEquals("meta_active", searchResponse.getResults().get(0).getSource().get("guid"));
    // Query against all indices. The child alert has no actual attached meta alerts, and should
    // be returned on its own.
    searchResponse = metaDao.search(new SearchRequest() {

        {
            setQuery("(ip_src_addr:192.168.1.3 AND ip_src_port:8008)" + " OR (alert.ip_src_addr:192.168.1.3 AND alert.ip_src_port:8008)");
            setIndices(Collections.singletonList("*"));
            setFrom(0);
            setSize(1);
            setSort(Collections.singletonList(new SortField() {

                {
                    setField(Constants.GUID);
                }
            }));
        }
    });
    // Nested query should match a plain alert
    Assert.assertEquals(1, searchResponse.getTotal());
    Assert.assertEquals("message_2", searchResponse.getResults().get(0).getSource().get("guid"));
}

Example 3 with ElasticsearchDao

use of org.apache.metron.elasticsearch.dao.ElasticsearchDao in project metron by apache.

the class ElasticsearchMetaAlertIntegrationTest method shouldHidesAlertsOnGroup.

@Test
public void shouldHidesAlertsOnGroup() throws Exception {
    // Load alerts
    List<Map<String, Object>> alerts = buildAlerts(2);
    alerts.get(0).put(METAALERT_FIELD, Collections.singletonList("meta_active"));
    alerts.get(0).put("ip_src_addr", "192.168.1.1");
    alerts.get(0).put("score_field", 1);
    alerts.get(1).put("ip_src_addr", "192.168.1.1");
    alerts.get(1).put("score_field", 10);
    elasticsearchAdd(alerts, INDEX, SENSOR_NAME);
    // Put the nested type into the test index, so that it'll match appropriately
    ((ElasticsearchDao) esDao).getClient().admin().indices().preparePutMapping(INDEX).setType("test_doc").setSource(nestedAlertMapping).get();
    // Don't need any meta alerts to actually exist, since we've populated the field on the alerts.
    // Verify load was successful
    findCreatedDocs(Arrays.asList(new GetRequest("message_0", SENSOR_NAME), new GetRequest("message_1", SENSOR_NAME)));
    // Build our group request
    Group searchGroup = new Group();
    searchGroup.setField("ip_src_addr");
    List<Group> groupList = new ArrayList<>();
    groupList.add(searchGroup);
    GroupResponse groupResponse = metaDao.group(new GroupRequest() {

        {
            setQuery("ip_src_addr:192.168.1.1");
            setIndices(Collections.singletonList("*"));
            setScoreField("score_field");
            setGroups(groupList);
        }
    });
    // Should only return the standalone alert in the group
    GroupResult result = groupResponse.getGroupResults().get(0);
    Assert.assertEquals(1, result.getTotal());
    Assert.assertEquals("192.168.1.1", result.getKey());
    // No delta, since no ops happen
    Assert.assertEquals(10.0d, result.getScore(), 0.0d);
}

Example 4 with ElasticsearchDao

use of org.apache.metron.elasticsearch.dao.ElasticsearchDao in project metron by apache.

the class ElasticsearchSearchIntegrationTest method createDao.

@Override
protected IndexDao createDao() throws Exception {
    AccessConfig config = new AccessConfig();
    config.setMaxSearchResults(100);
    config.setMaxSearchGroups(100);
    config.setGlobalConfigSupplier(() -> new HashMap<String, Object>() {

        {
            put("es.clustername", "metron");
            put("es.port", "9300");
            put("es.ip", "localhost");
            put("es.date.format", dateFormat);
        }
    });
    IndexDao dao = new ElasticsearchDao();
    dao.init(config);
    return dao;
}

Example 5 with ElasticsearchDao

use of org.apache.metron.elasticsearch.dao.ElasticsearchDao in project metron by apache.

the class ElasticsearchUpdateIntegrationTest method setup.

@BeforeClass
public static void setup() throws Exception {
    Configuration config = HBaseConfiguration.create();
    MockHBaseTableProvider tableProvider = new MockHBaseTableProvider();
    tableProvider.addToCache(TABLE_NAME, CF);
    table = (MockHTable) tableProvider.getTable(config, TABLE_NAME);
    // setup the client
    es = new ElasticSearchComponent.Builder().withHttpPort(9211).withIndexDir(new File(indexDir)).build();
    es.start();
    hbaseDao = new HBaseDao();
    AccessConfig accessConfig = new AccessConfig();
    accessConfig.setTableProvider(tableProvider);
    Map<String, Object> globalConfig = new HashMap<String, Object>() {

        {
            put("es.clustername", "metron");
            put("es.port", "9300");
            put("es.ip", "localhost");
            put("es.date.format", dateFormat);
            put(HBaseDao.HBASE_TABLE, TABLE_NAME);
            put(HBaseDao.HBASE_CF, CF);
        }
    };
    accessConfig.setGlobalConfigSupplier(() -> globalConfig);
    esDao = new ElasticsearchDao();
    dao = new MultiIndexDao(hbaseDao, esDao);
    dao.init(accessConfig);
}