Examples with IndexingConfigurations org.apache.metron.common.configuration.IndexingConfigurations used on opensource projects

Example 31 with IndexingConfigurations

use of org.apache.metron.common.configuration.IndexingConfigurations in project metron by apache.

the class HdfsWriterTest method testGetHdfsPathFormatVariable.

@Test
@SuppressWarnings("unchecked")
public void testGetHdfsPathFormatVariable() {
    IndexingConfigurations indexingConfig = new IndexingConfigurations();
    WriterConfiguration config = new IndexingWriterConfiguration(WRITER_NAME, indexingConfig);
    HdfsWriter writer = new HdfsWriter().withFileNameFormat(testFormat);
    writer.init(new HashMap<String, String>(), config);
    writer.initFileNameFormat(createTopologyContext());
    JSONObject message = new JSONObject();
    message.put("test.key", "test.value");
    message.put("test.key.2", "test.value.2");
    message.put("test.key.3", "test.value.3");
    Object result = writer.getHdfsPathExtension(SENSOR_NAME, "FORMAT('%s/%s/%s', test.key, test.key.2, test.key.3)", message);
    writer.close();
    assertEquals("test.value/test.value.2/test.value.3", result);
}

Example 32 with IndexingConfigurations

use of org.apache.metron.common.configuration.IndexingConfigurations in project metron by apache.

the class HdfsWriterTest method testGetSourceHandlerOpenFilesMax.

@Test
public void testGetSourceHandlerOpenFilesMax() throws IOException {
    int maxFiles = 2;
    IndexingConfigurations indexingConfig = new IndexingConfigurations();
    WriterConfiguration config = new IndexingWriterConfiguration(WRITER_NAME, indexingConfig);
    HdfsWriter writer = new HdfsWriter().withFileNameFormat(testFormat).withMaxOpenFiles(maxFiles);
    writer.init(new HashMap<String, String>(), config);
    writer.initFileNameFormat(createTopologyContext());
    for (int i = 0; i < maxFiles; i++) {
        writer.getSourceHandler(SENSOR_NAME, Integer.toString(i), null);
    }
    writer.close();
}

Example 33 with IndexingConfigurations

use of org.apache.metron.common.configuration.IndexingConfigurations in project metron by apache.

the class HdfsWriterTest method testWriteNoOutputFunction.

@Test
@SuppressWarnings("unchecked")
public void testWriteNoOutputFunction() throws Exception {
    FileNameFormat format = new DefaultFileNameFormat().withPath(folder.toString()).withExtension(".json").withPrefix("prefix-");
    HdfsWriter writer = new HdfsWriter().withFileNameFormat(format);
    IndexingConfigurations indexingConfig = new IndexingConfigurations();
    WriterConfiguration config = new IndexingWriterConfiguration(WRITER_NAME, indexingConfig);
    writer.init(new HashMap<String, String>(), config);
    writer.initFileNameFormat(createTopologyContext());
    JSONObject message = new JSONObject();
    message.put("test.key", "test.value");
    message.put("test.key2", "test.value2");
    JSONObject message2 = new JSONObject();
    message2.put("test.key", "test.value3");
    message2.put("test.key2", "test.value2");
    List<BulkMessage<JSONObject>> messages = new ArrayList<BulkMessage<JSONObject>>() {

        {
            add(new BulkMessage("message1", message));
            add(new BulkMessage("message2", message2));
        }
    };
    writer.write(SENSOR_NAME, config, messages);
    writer.close();
    ArrayList<String> expected = new ArrayList<>();
    expected.add(message.toJSONString());
    expected.add(message2.toJSONString());
    Collections.sort(expected);
    // Default to just putting it in the base folder + the sensor name
    File outputFolder = new File(folder.getAbsolutePath() + "/" + SENSOR_NAME);
    assertTrue(outputFolder.exists() && outputFolder.isDirectory());
    assertEquals(1, outputFolder.listFiles().length);
    for (File file : outputFolder.listFiles()) {
        List<String> lines = Files.readAllLines(file.toPath());
        Collections.sort(lines);
        assertEquals(expected, lines);
    }
}

Example 34 with IndexingConfigurations

use of org.apache.metron.common.configuration.IndexingConfigurations in project metron by apache.

the class SensorIndexingConfigServiceImplTest method getAllIndicesWithParsersAndIndexConfigs.

@Test
public void getAllIndicesWithParsersAndIndexConfigs() throws RestException {
    ParserConfigurations parserConfiguration = mock(ParserConfigurations.class);
    when(parserConfiguration.getTypes()).thenReturn(ImmutableList.of("bro", "yaf"));
    IndexingConfigurations indexingConfiguration = mock(IndexingConfigurations.class);
    when(indexingConfiguration.getTypes()).thenReturn(ImmutableList.of("bro", "snort", "squid"));
    when(indexingConfiguration.getIndex(eq("bro"), eq("elasticsearch"))).thenReturn("renamed_bro");
    when(indexingConfiguration.getIndex(eq("snort"), eq("elasticsearch"))).thenReturn("snort");
    when(indexingConfiguration.getIndex(eq("yaf"), eq("elasticsearch"))).thenReturn(null);
    when(indexingConfiguration.isEnabled(eq("snort"), eq("elasticsearch"))).thenReturn(true);
    when(indexingConfiguration.isEnabled(eq("bro"), eq("elasticsearch"))).thenReturn(true);
    when(indexingConfiguration.isEnabled(eq("yaf"), eq("elasticsearch"))).thenReturn(true);
    when(indexingConfiguration.isEnabled(eq("squid"), eq("elasticsearch"))).thenReturn(false);
    when(cache.get(eq(ParserConfigurations.class))).thenReturn(parserConfiguration);
    when(cache.get(eq(IndexingConfigurations.class))).thenReturn(indexingConfiguration);
    List<String> indices = new ArrayList<String>();
    Iterables.addAll(indices, sensorIndexingConfigService.getAllIndices("elasticsearch"));
    assertEquals(3, indices.size());
    assertTrue(indices.contains("renamed_bro"));
    assertTrue(indices.contains("snort"));
    assertTrue(indices.contains("yaf"));
}

Example 35 with IndexingConfigurations

use of org.apache.metron.common.configuration.IndexingConfigurations in project metron by apache.

the class SensorIndexingConfigServiceImplTest method getAllIndicesWithOnlyParsers.

@Test
public void getAllIndicesWithOnlyParsers() throws RestException {
    ParserConfigurations parserConfiguration = mock(ParserConfigurations.class);
    when(parserConfiguration.getTypes()).thenReturn(ImmutableList.of("bro", "snort"));
    IndexingConfigurations indexingConfiguration = mock(IndexingConfigurations.class);
    when(indexingConfiguration.getTypes()).thenReturn(Collections.emptyList());
    when(indexingConfiguration.getIndex(eq("bro"), eq("elasticsearch"))).thenReturn(null);
    when(indexingConfiguration.getIndex(eq("snort"), eq("elasticsearch"))).thenReturn(null);
    when(indexingConfiguration.isEnabled(eq("snort"), eq("elasticsearch"))).thenReturn(true);
    when(indexingConfiguration.isEnabled(eq("bro"), eq("elasticsearch"))).thenReturn(true);
    when(cache.get(eq(ParserConfigurations.class))).thenReturn(parserConfiguration);
    when(cache.get(eq(IndexingConfigurations.class))).thenReturn(indexingConfiguration);
    List<String> indices = new ArrayList<String>();
    Iterables.addAll(indices, sensorIndexingConfigService.getAllIndices("elasticsearch"));
    assertEquals(2, indices.size());
    assertTrue(indices.contains("bro"));
    assertTrue(indices.contains("snort"));
}