Examples with IndexDao org.apache.metron.indexing.dao.IndexDao used on opensource projects

Example 1 with IndexDao

use of org.apache.metron.indexing.dao.IndexDao in project metron by apache.

the class IndexConfig method indexDao.

@Bean
public IndexDao indexDao() {
    try {
        String hbaseProviderImpl = environment.getProperty(MetronRestConstants.INDEX_HBASE_TABLE_PROVIDER_IMPL, String.class, null);
        String indexDaoImpl = environment.getProperty(MetronRestConstants.INDEX_DAO_IMPL, String.class, null);
        int searchMaxResults = environment.getProperty(MetronRestConstants.SEARCH_MAX_RESULTS, Integer.class, 1000);
        int searchMaxGroups = environment.getProperty(MetronRestConstants.SEARCH_MAX_GROUPS, Integer.class, 1000);
        String metaDaoImpl = environment.getProperty(MetronRestConstants.META_DAO_IMPL, String.class, null);
        String metaDaoSort = environment.getProperty(MetronRestConstants.META_DAO_SORT, String.class, null);
        AccessConfig config = new AccessConfig();
        config.setMaxSearchResults(searchMaxResults);
        config.setMaxSearchGroups(searchMaxGroups);
        config.setGlobalConfigSupplier(() -> {
            try {
                return globalConfigService.get();
            } catch (RestException e) {
                throw new IllegalStateException("Unable to retrieve the global config.", e);
            }
        });
        config.setIndexSupplier(IndexingCacheUtil.getIndexLookupFunction(cache, environment.getProperty(INDEX_WRITER_NAME)));
        config.setTableProvider(TableProvider.create(hbaseProviderImpl, () -> new HTableProvider()));
        config.setKerberosEnabled(environment.getProperty(MetronRestConstants.KERBEROS_ENABLED_SPRING_PROPERTY, Boolean.class, false));
        if (indexDaoImpl == null) {
            throw new IllegalStateException("You must provide an index DAO implementation via the " + INDEX_DAO_IMPL + " config");
        }
        IndexDao indexDao = IndexDaoFactory.combine(IndexDaoFactory.create(indexDaoImpl, config));
        if (indexDao == null) {
            throw new IllegalStateException("IndexDao is unable to be created.");
        }
        if (metaDaoImpl == null) {
            // We're not using meta alerts.
            return indexDao;
        }
        // Create the meta alert dao and wrap it around the index dao.
        MetaAlertDao ret = (MetaAlertDao) IndexDaoFactory.create(metaDaoImpl, config).get(0);
        ret.init(indexDao, Optional.ofNullable(metaDaoSort));
        return ret;
    } catch (RuntimeException re) {
        throw re;
    } catch (Exception e) {
        throw new IllegalStateException("Unable to create index DAO: " + e.getMessage(), e);
    }
}

Example 2 with IndexDao

use of org.apache.metron.indexing.dao.IndexDao in project metron by apache.

the class ElasticsearchSearchIntegrationTest method createDao.

@Override
protected IndexDao createDao() throws Exception {
    AccessConfig config = new AccessConfig();
    config.setMaxSearchResults(100);
    config.setMaxSearchGroups(100);
    config.setGlobalConfigSupplier(() -> new HashMap<String, Object>() {

        {
            put("es.clustername", "metron");
            put("es.port", "9300");
            put("es.ip", "localhost");
            put("es.date.format", dateFormat);
        }
    });
    IndexDao dao = new ElasticsearchDao();
    dao.init(config);
    return dao;
}

Example 3 with IndexDao

use of org.apache.metron.indexing.dao.IndexDao in project metron by apache.

the class ElasticsearchMetaAlertDao method init.

/**
 * Initializes this implementation by setting the supplied IndexDao and also setting a separate
 *     ElasticsearchDao.
 * This is needed for some specific Elasticsearch functions (looking up an index from a GUID for
 *     example).
 * @param indexDao The DAO to wrap for our queries
 * @param threatSort The summary aggregation of the child threat triage scores used
 *                   as the overall threat triage score for the metaalert. This
 *                   can be either max, min, average, count, median, or sum.
 */
@Override
public void init(IndexDao indexDao, Optional<String> threatSort) {
    if (indexDao instanceof MultiIndexDao) {
        this.indexDao = indexDao;
        MultiIndexDao multiIndexDao = (MultiIndexDao) indexDao;
        for (IndexDao childDao : multiIndexDao.getIndices()) {
            if (childDao instanceof ElasticsearchDao) {
                this.elasticsearchDao = (ElasticsearchDao) childDao;
            }
        }
    } else if (indexDao instanceof ElasticsearchDao) {
        this.indexDao = indexDao;
        this.elasticsearchDao = (ElasticsearchDao) indexDao;
    } else {
        throw new IllegalArgumentException("Need an ElasticsearchDao when using ElasticsearchMetaAlertDao");
    }
    if (threatSort.isPresent()) {
        this.threatSort = threatSort.get();
    }
    Supplier<Map<String, Object>> globalConfigSupplier = () -> new HashMap<>();
    if (elasticsearchDao != null && elasticsearchDao.getAccessConfig() != null) {
        globalConfigSupplier = elasticsearchDao.getAccessConfig().getGlobalConfigSupplier();
    }
    MetaAlertConfig config = new MetaAlertConfig(metaAlertsIndex, this.threatSort, globalConfigSupplier) {

        @Override
        protected String getDefaultThreatTriageField() {
            return THREAT_TRIAGE_FIELD;
        }

        @Override
        protected String getDefaultSourceTypeField() {
            return SOURCE_TYPE_FIELD;
        }
    };
    this.metaAlertSearchDao = new ElasticsearchMetaAlertSearchDao(elasticsearchDao, config, pageSize);
    this.metaAlertRetrieveLatestDao = new ElasticsearchMetaAlertRetrieveLatestDao(indexDao);
    this.metaAlertUpdateDao = new ElasticsearchMetaAlertUpdateDao(elasticsearchDao, metaAlertRetrieveLatestDao, config, pageSize);
}

Example 4 with IndexDao

use of org.apache.metron.indexing.dao.IndexDao in project metron by apache.

the class ElasticsearchMetaAlertDaoTest method testInvalidInit.

@Test
public void testInvalidInit() {
    IndexDao dao = new IndexDao() {

        @Override
        public SearchResponse search(SearchRequest searchRequest) {
            return null;
        }

        @Override
        public GroupResponse group(GroupRequest groupRequest) {
            return null;
        }

        @Override
        public void init(AccessConfig config) {
        }

        @Override
        public Document getLatest(String guid, String sensorType) {
            return null;
        }

        @Override
        public Iterable<Document> getAllLatest(List<GetRequest> getRequests) {
            return null;
        }

        @Override
        public Document update(Document update, Optional<String> index) {
            return update;
        }

        @Override
        public Map<Document, Optional<String>> batchUpdate(Map<Document, Optional<String>> updates) {
            return updates;
        }

        @Override
        public Map<String, FieldType> getColumnMetadata(List<String> indices) {
            return null;
        }

        @Override
        public Document addCommentToAlert(CommentAddRemoveRequest request) {
            return null;
        }

        @Override
        public Document removeCommentFromAlert(CommentAddRemoveRequest request) {
            return null;
        }

        @Override
        public Document addCommentToAlert(CommentAddRemoveRequest request, Document latest) {
            return null;
        }

        @Override
        public Document removeCommentFromAlert(CommentAddRemoveRequest request, Document latest) {
            return null;
        }
    };
    ElasticsearchMetaAlertDao metaAlertDao = new ElasticsearchMetaAlertDao();
    assertThrows(IllegalArgumentException.class, () -> metaAlertDao.init(dao));
}

Example 5 with IndexDao

use of org.apache.metron.indexing.dao.IndexDao in project metron by apache.

the class SolrMetaAlertDao method init.

/**
 * Initializes this implementation by setting the supplied IndexDao and also setting a separate SolrDao.
 * This is needed for some specific Solr functions (looking up an index from a GUID for example).
 * @param indexDao The DAO to wrap for our queries
 * @param threatSort The summary aggregation of the child threat triage scores used
 *                   as the overall threat triage score for the metaalert. This
 *                   can be either max, min, average, count, median, or sum.
 */
@Override
public void init(IndexDao indexDao, Optional<String> threatSort) {
    if (indexDao instanceof MultiIndexDao) {
        this.indexDao = indexDao;
        MultiIndexDao multiIndexDao = (MultiIndexDao) indexDao;
        for (IndexDao childDao : multiIndexDao.getIndices()) {
            if (childDao instanceof SolrDao) {
                this.solrDao = (SolrDao) childDao;
            }
        }
    } else if (indexDao instanceof SolrDao) {
        this.indexDao = indexDao;
        this.solrDao = (SolrDao) indexDao;
    } else {
        throw new IllegalArgumentException("Need a SolrDao when using SolrMetaAlertDao");
    }
    Supplier<Map<String, Object>> globalConfigSupplier = () -> new HashMap<>();
    if (metaAlertSearchDao != null && metaAlertSearchDao.solrSearchDao != null && metaAlertSearchDao.solrSearchDao.getAccessConfig() != null) {
        globalConfigSupplier = metaAlertSearchDao.solrSearchDao.getAccessConfig().getGlobalConfigSupplier();
    }
    MetaAlertConfig config = new MetaAlertConfig(metaAlertsCollection, this.threatSort, globalConfigSupplier) {

        @Override
        protected String getDefaultThreatTriageField() {
            return MetaAlertConstants.THREAT_FIELD_DEFAULT.replace(':', '.');
        }

        @Override
        protected String getDefaultSourceTypeField() {
            return Constants.SENSOR_TYPE;
        }
    };
    SolrClient solrClient = SolrClientFactory.create(globalConfigSupplier.get());
    this.metaAlertSearchDao = new SolrMetaAlertSearchDao(solrClient, solrDao.getSolrSearchDao(), config);
    this.metaAlertRetrieveLatestDao = new SolrMetaAlertRetrieveLatestDao(solrClient, solrDao);
    this.metaAlertUpdateDao = new SolrMetaAlertUpdateDao(solrClient, solrDao, metaAlertSearchDao, metaAlertRetrieveLatestDao, config);
    if (threatSort.isPresent()) {
        this.threatSort = threatSort.get();
    }
}