Examples with IndexingWriterConfiguration org.apache.metron.common.configuration.writer.IndexingWriterConfiguration used on opensource projects

Example 11 with IndexingWriterConfiguration

use of org.apache.metron.common.configuration.writer.IndexingWriterConfiguration in project metron by apache.

the class IndexingWriterConfigurationTest method testDefaultBatchSize.

@Test
public void testDefaultBatchSize() {
    IndexingWriterConfiguration config = new IndexingWriterConfiguration("hdfs", new IndexingConfigurations());
    Assert.assertEquals(1, config.getBatchSize("foo"));
}

Example 12 with IndexingWriterConfiguration

use of org.apache.metron.common.configuration.writer.IndexingWriterConfiguration in project metron by apache.

the class SolrWriterTest method testWriter.

@Test
public void testWriter() throws Exception {
    IndexingConfigurations configurations = SampleUtil.getSampleIndexingConfigs();
    JSONObject message1 = new JSONObject();
    message1.put("intField", 100);
    message1.put("doubleField", 100.0);
    JSONObject message2 = new JSONObject();
    message2.put("intField", 200);
    message2.put("doubleField", 200.0);
    List<JSONObject> messages = new ArrayList<>();
    messages.add(message1);
    messages.add(message2);
    String collection = "metron";
    MetronSolrClient solr = Mockito.mock(MetronSolrClient.class);
    SolrWriter writer = new SolrWriter().withMetronSolrClient(solr);
    writer.init(null, null, new IndexingWriterConfiguration("solr", configurations));
    verify(solr, times(1)).createCollection(collection, 1, 1);
    verify(solr, times(1)).setDefaultCollection(collection);
    collection = "metron2";
    int numShards = 4;
    int replicationFactor = 2;
    Map<String, Object> globalConfig = configurations.getGlobalConfig();
    globalConfig.put("solr.collection", collection);
    globalConfig.put("solr.numShards", numShards);
    globalConfig.put("solr.replicationFactor", replicationFactor);
    configurations.updateGlobalConfig(globalConfig);
    writer = new SolrWriter().withMetronSolrClient(solr);
    writer.init(null, null, new IndexingWriterConfiguration("solr", configurations));
    verify(solr, times(1)).createCollection(collection, numShards, replicationFactor);
    verify(solr, times(1)).setDefaultCollection(collection);
    writer.write("test", new IndexingWriterConfiguration("solr", configurations), new ArrayList<>(), messages);
    verify(solr, times(1)).add(argThat(new SolrInputDocumentMatcher(message1.toJSONString().hashCode(), "test", 100, 100.0)));
    verify(solr, times(1)).add(argThat(new SolrInputDocumentMatcher(message2.toJSONString().hashCode(), "test", 200, 200.0)));
    verify(solr, times(0)).commit(collection);
    writer = new SolrWriter().withMetronSolrClient(solr).withShouldCommit(true);
    writer.init(null, null, new IndexingWriterConfiguration("solr", configurations));
    writer.write("test", new IndexingWriterConfiguration("solr", configurations), new ArrayList<>(), messages);
    verify(solr, times(2)).add(argThat(new SolrInputDocumentMatcher(message1.toJSONString().hashCode(), "test", 100, 100.0)));
    verify(solr, times(2)).add(argThat(new SolrInputDocumentMatcher(message2.toJSONString().hashCode(), "test", 200, 200.0)));
    verify(solr, times(1)).commit(collection);
}

Example 13 with IndexingWriterConfiguration

use of org.apache.metron.common.configuration.writer.IndexingWriterConfiguration in project metron by apache.

the class BulkMessageWriterBolt method getComponentConfiguration.

/**
 * This method is called by TopologyBuilder.createTopology() to obtain topology and
 * bolt specific configuration parameters.  We use it primarily to configure how often
 * a tick tuple will be sent to our bolt.
 * @return conf topology and bolt specific configuration parameters
 */
@Override
public Map<String, Object> getComponentConfiguration() {
    // This is called long before prepare(), so do some of the same stuff as prepare() does,
    // to get the valid WriterConfiguration.  But don't store any non-serializable objects,
    // else Storm will throw a runtime error.
    Function<WriterConfiguration, WriterConfiguration> configurationXform;
    if (bulkMessageWriter instanceof WriterToBulkWriter) {
        configurationXform = WriterToBulkWriter.TRANSFORMATION;
    } else {
        configurationXform = x -> x;
    }
    WriterConfiguration writerconf = configurationXform.apply(new IndexingWriterConfiguration(bulkMessageWriter.getName(), getConfigurations()));
    BatchTimeoutHelper timeoutHelper = new BatchTimeoutHelper(writerconf::getAllConfiguredTimeouts, batchTimeoutDivisor);
    this.requestedTickFreqSecs = timeoutHelper.getRecommendedTickInterval();
    // And while we've got BatchTimeoutHelper handy, capture the defaultBatchTimeout for writerComponent.
    this.defaultBatchTimeout = timeoutHelper.getDefaultBatchTimeout();
    Map<String, Object> conf = super.getComponentConfiguration();
    if (conf == null) {
        conf = new HashMap<String, Object>();
    }
    conf.put(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, requestedTickFreqSecs);
    LOG.info("Requesting " + Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS + " set to " + Integer.toString(requestedTickFreqSecs));
    return conf;
}

Example 14 with IndexingWriterConfiguration

use of org.apache.metron.common.configuration.writer.IndexingWriterConfiguration in project metron by apache.

the class BulkMessageWriterBolt method execute.

@SuppressWarnings("unchecked")
@Override
public void execute(Tuple tuple) {
    if (isTick(tuple)) {
        try {
            if (!(bulkMessageWriter instanceof WriterToBulkWriter)) {
                // WriterToBulkWriter doesn't allow batching, so no need to flush on Tick.
                LOG.debug("Flushing message queues older than their batchTimeouts");
                writerComponent.flushTimeouts(bulkMessageWriter, configurationTransformation.apply(new IndexingWriterConfiguration(bulkMessageWriter.getName(), getConfigurations())), messageGetStrategy);
            }
        } catch (Exception e) {
            throw new RuntimeException("This should have been caught in the writerComponent.  If you see this, file a JIRA", e);
        } finally {
            collector.ack(tuple);
        }
        return;
    }
    try {
        JSONObject message = (JSONObject) messageGetStrategy.get(tuple);
        String sensorType = MessageUtils.getSensorType(message);
        LOG.trace("Writing enrichment message: {}", message);
        WriterConfiguration writerConfiguration = configurationTransformation.apply(new IndexingWriterConfiguration(bulkMessageWriter.getName(), getConfigurations()));
        if (writerConfiguration.isDefault(sensorType)) {
            // want to warn, but not fail the tuple
            collector.reportError(new Exception("WARNING: Default and (likely) unoptimized writer config used for " + bulkMessageWriter.getName() + " writer and sensor " + sensorType));
        }
        writerComponent.write(sensorType, tuple, message, bulkMessageWriter, writerConfiguration, messageGetStrategy);
    } catch (Exception e) {
        throw new RuntimeException("This should have been caught in the writerComponent.  If you see this, file a JIRA", e);
    }
}

Example 15 with IndexingWriterConfiguration

use of org.apache.metron.common.configuration.writer.IndexingWriterConfiguration in project metron by apache.

the class HdfsWriterTest method testGetHdfsPathFormatVariable.

@Test
@SuppressWarnings("unchecked")
public void testGetHdfsPathFormatVariable() {
    IndexingConfigurations indexingConfig = new IndexingConfigurations();
    WriterConfiguration config = new IndexingWriterConfiguration(WRITER_NAME, indexingConfig);
    HdfsWriter writer = new HdfsWriter().withFileNameFormat(testFormat);
    writer.init(new HashMap<String, String>(), createTopologyContext(), config);
    JSONObject message = new JSONObject();
    message.put("test.key", "test.value");
    message.put("test.key.2", "test.value.2");
    message.put("test.key.3", "test.value.3");
    Object result = writer.getHdfsPathExtension(SENSOR_NAME, "FORMAT('%s/%s/%s', test.key, test.key.2, test.key.3)", message);
    writer.close();
    Assert.assertEquals("test.value/test.value.2/test.value.3", result);
}