Examples with ElasticSearchComponent org.apache.metron.elasticsearch.integration.components.ElasticSearchComponent used on opensource projects

Example 1 with ElasticSearchComponent

use of org.apache.metron.elasticsearch.integration.components.ElasticSearchComponent in project metron by apache.

the class ElasticsearchIndexingIntegrationTest method getProcessor.

@Override
public Processor<List<Map<String, Object>>> getProcessor(final List<byte[]> inputMessages) {
    return new Processor<List<Map<String, Object>>>() {

        List<Map<String, Object>> docs = null;

        List<byte[]> errors = null;

        final AtomicInteger missCount = new AtomicInteger(0);

        @Override
        public ReadinessState process(ComponentRunner runner) {
            ElasticSearchComponent elasticSearchComponent = runner.getComponent("search", ElasticSearchComponent.class);
            KafkaComponent kafkaComponent = runner.getComponent("kafka", KafkaComponent.class);
            if (elasticSearchComponent.hasIndex(index)) {
                try {
                    docs = elasticSearchComponent.getAllIndexedDocs(index, testSensorType + "_doc");
                } catch (IOException e) {
                    throw new IllegalStateException("Unable to retrieve indexed documents.", e);
                }
                if (docs.size() < inputMessages.size()) {
                    errors = kafkaComponent.readMessages(ERROR_TOPIC);
                    if (errors.size() > 0 && errors.size() + docs.size() == inputMessages.size()) {
                        return ReadinessState.READY;
                    }
                    return ReadinessState.NOT_READY;
                } else {
                    return ReadinessState.READY;
                }
            } else {
                return ReadinessState.NOT_READY;
            }
        }

        @Override
        public ProcessorResult<List<Map<String, Object>>> getResult() {
            ProcessorResult.Builder<List<Map<String, Object>>> builder = new ProcessorResult.Builder();
            return builder.withResult(docs).withProcessErrors(errors).build();
        }
    };
}

Example 2 with ElasticSearchComponent

use of org.apache.metron.elasticsearch.integration.components.ElasticSearchComponent in project metron by apache.

the class ElasticsearchSearchIntegrationTest method loadTestData.

@Override
protected void loadTestData() throws ParseException, IOException, ExecutionException, InterruptedException {
    ElasticSearchComponent es = (ElasticSearchComponent) indexComponent;
    es.getClient().admin().indices().prepareCreate("bro_index_2017.01.01.01").addMapping("bro_doc", broTypeMappings).addMapping("bro_doc_default", broDefaultStringMappings).get();
    es.getClient().admin().indices().prepareCreate("snort_index_2017.01.01.02").addMapping("snort_doc", snortTypeMappings).get();
    BulkRequestBuilder bulkRequest = es.getClient().prepareBulk().setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL);
    JSONArray broArray = (JSONArray) new JSONParser().parse(broData);
    for (Object o : broArray) {
        JSONObject jsonObject = (JSONObject) o;
        IndexRequestBuilder indexRequestBuilder = es.getClient().prepareIndex("bro_index_2017.01.01.01", "bro_doc");
        indexRequestBuilder = indexRequestBuilder.setId((String) jsonObject.get("guid"));
        indexRequestBuilder = indexRequestBuilder.setSource(jsonObject.toJSONString());
        indexRequestBuilder = indexRequestBuilder.setTimestamp(jsonObject.get("timestamp").toString());
        bulkRequest.add(indexRequestBuilder);
    }
    JSONArray snortArray = (JSONArray) new JSONParser().parse(snortData);
    for (Object o : snortArray) {
        JSONObject jsonObject = (JSONObject) o;
        IndexRequestBuilder indexRequestBuilder = es.getClient().prepareIndex("snort_index_2017.01.01.02", "snort_doc");
        indexRequestBuilder = indexRequestBuilder.setId((String) jsonObject.get("guid"));
        indexRequestBuilder = indexRequestBuilder.setSource(jsonObject.toJSONString());
        indexRequestBuilder = indexRequestBuilder.setTimestamp(jsonObject.get("timestamp").toString());
        bulkRequest.add(indexRequestBuilder);
    }
    BulkResponse bulkResponse = bulkRequest.execute().actionGet();
    if (bulkResponse.hasFailures()) {
        throw new RuntimeException("Failed to index test data");
    }
}