Examples with IndexingConfigurations org.apache.metron.common.configuration.IndexingConfigurations used on opensource projects

Example 16 with IndexingConfigurations

use of org.apache.metron.common.configuration.IndexingConfigurations in project metron by apache.

the class SensorIndexingConfigServiceImplTest method getAllTypesShouldProperlyReturnTypes.

@Test
public void getAllTypesShouldProperlyReturnTypes() throws Exception {
    IndexingConfigurations configs = new IndexingConfigurations() {

        @Override
        public Map<String, Object> getConfigurations() {
            return ImmutableMap.of(IndexingConfigurations.getKey("bro"), new HashMap<>(), IndexingConfigurations.getKey("squid"), new HashMap<>());
        }
    };
    when(cache.get(eq(IndexingConfigurations.class))).thenReturn(configs);
    assertEquals(new ArrayList() {

        {
            add("bro");
            add("squid");
        }
    }, sensorIndexingConfigService.getAllTypes());
}

Example 17 with IndexingConfigurations

use of org.apache.metron.common.configuration.IndexingConfigurations in project metron by apache.

the class SensorIndexingConfigServiceImplTest method getAllIndicesWithParsersAndIndexConfigs.

@Test
public void getAllIndicesWithParsersAndIndexConfigs() throws RestException {
    ParserConfigurations parserConfiguration = mock(ParserConfigurations.class);
    when(parserConfiguration.getTypes()).thenReturn(ImmutableList.of("bro", "yaf"));
    IndexingConfigurations indexingConfiguration = mock(IndexingConfigurations.class);
    when(indexingConfiguration.getTypes()).thenReturn(ImmutableList.of("bro", "snort", "squid"));
    when(indexingConfiguration.getIndex(eq("bro"), eq("elasticsearch"))).thenReturn("renamed_bro");
    when(indexingConfiguration.getIndex(eq("snort"), eq("elasticsearch"))).thenReturn("snort");
    when(indexingConfiguration.getIndex(eq("yaf"), eq("elasticsearch"))).thenReturn(null);
    when(indexingConfiguration.isEnabled(eq("snort"), eq("elasticsearch"))).thenReturn(true);
    when(indexingConfiguration.isEnabled(eq("bro"), eq("elasticsearch"))).thenReturn(true);
    when(indexingConfiguration.isEnabled(eq("yaf"), eq("elasticsearch"))).thenReturn(true);
    when(indexingConfiguration.isEnabled(eq("squid"), eq("elasticsearch"))).thenReturn(false);
    when(cache.get(eq(ParserConfigurations.class))).thenReturn(parserConfiguration);
    when(cache.get(eq(IndexingConfigurations.class))).thenReturn(indexingConfiguration);
    List<String> indices = new ArrayList<String>();
    Iterables.addAll(indices, sensorIndexingConfigService.getAllIndices("elasticsearch"));
    Assert.assertEquals(3, indices.size());
    Assert.assertTrue(indices.contains("renamed_bro"));
    Assert.assertTrue(indices.contains("snort"));
    Assert.assertTrue(indices.contains("yaf"));
}

Example 18 with IndexingConfigurations

use of org.apache.metron.common.configuration.IndexingConfigurations in project metron by apache.

the class IndexingWriterConfigurationTest method testDefaultBatchSize.

@Test
public void testDefaultBatchSize() {
    IndexingWriterConfiguration config = new IndexingWriterConfiguration("hdfs", new IndexingConfigurations());
    Assert.assertEquals(1, config.getBatchSize("foo"));
}

Example 19 with IndexingConfigurations

use of org.apache.metron.common.configuration.IndexingConfigurations in project metron by apache.

the class SolrWriterTest method testWriter.

@Test
public void testWriter() throws Exception {
    IndexingConfigurations configurations = SampleUtil.getSampleIndexingConfigs();
    JSONObject message1 = new JSONObject();
    message1.put("intField", 100);
    message1.put("doubleField", 100.0);
    JSONObject message2 = new JSONObject();
    message2.put("intField", 200);
    message2.put("doubleField", 200.0);
    List<JSONObject> messages = new ArrayList<>();
    messages.add(message1);
    messages.add(message2);
    String collection = "metron";
    MetronSolrClient solr = Mockito.mock(MetronSolrClient.class);
    SolrWriter writer = new SolrWriter().withMetronSolrClient(solr);
    writer.init(null, null, new IndexingWriterConfiguration("solr", configurations));
    verify(solr, times(1)).createCollection(collection, 1, 1);
    verify(solr, times(1)).setDefaultCollection(collection);
    collection = "metron2";
    int numShards = 4;
    int replicationFactor = 2;
    Map<String, Object> globalConfig = configurations.getGlobalConfig();
    globalConfig.put("solr.collection", collection);
    globalConfig.put("solr.numShards", numShards);
    globalConfig.put("solr.replicationFactor", replicationFactor);
    configurations.updateGlobalConfig(globalConfig);
    writer = new SolrWriter().withMetronSolrClient(solr);
    writer.init(null, null, new IndexingWriterConfiguration("solr", configurations));
    verify(solr, times(1)).createCollection(collection, numShards, replicationFactor);
    verify(solr, times(1)).setDefaultCollection(collection);
    writer.write("test", new IndexingWriterConfiguration("solr", configurations), new ArrayList<>(), messages);
    verify(solr, times(1)).add(argThat(new SolrInputDocumentMatcher(message1.toJSONString().hashCode(), "test", 100, 100.0)));
    verify(solr, times(1)).add(argThat(new SolrInputDocumentMatcher(message2.toJSONString().hashCode(), "test", 200, 200.0)));
    verify(solr, times(0)).commit(collection);
    writer = new SolrWriter().withMetronSolrClient(solr).withShouldCommit(true);
    writer.init(null, null, new IndexingWriterConfiguration("solr", configurations));
    writer.write("test", new IndexingWriterConfiguration("solr", configurations), new ArrayList<>(), messages);
    verify(solr, times(2)).add(argThat(new SolrInputDocumentMatcher(message1.toJSONString().hashCode(), "test", 100, 100.0)));
    verify(solr, times(2)).add(argThat(new SolrInputDocumentMatcher(message2.toJSONString().hashCode(), "test", 200, 200.0)));
    verify(solr, times(1)).commit(collection);
}

Example 20 with IndexingConfigurations

use of org.apache.metron.common.configuration.IndexingConfigurations in project metron by apache.

the class HdfsWriterTest method testGetHdfsPathFormatVariable.

@Test
@SuppressWarnings("unchecked")
public void testGetHdfsPathFormatVariable() {
    IndexingConfigurations indexingConfig = new IndexingConfigurations();
    WriterConfiguration config = new IndexingWriterConfiguration(WRITER_NAME, indexingConfig);
    HdfsWriter writer = new HdfsWriter().withFileNameFormat(testFormat);
    writer.init(new HashMap<String, String>(), createTopologyContext(), config);
    JSONObject message = new JSONObject();
    message.put("test.key", "test.value");
    message.put("test.key.2", "test.value.2");
    message.put("test.key.3", "test.value.3");
    Object result = writer.getHdfsPathExtension(SENSOR_NAME, "FORMAT('%s/%s/%s', test.key, test.key.2, test.key.3)", message);
    writer.close();
    Assert.assertEquals("test.value/test.value.2/test.value.3", result);
}