Examples with ProfilerConfigurations org.apache.metron.common.configuration.profiler.ProfilerConfigurations used on opensource projects

Example 1 with ProfilerConfigurations

use of org.apache.metron.common.configuration.profiler.ProfilerConfigurations in project metron by apache.

the class ProfileBuilderBolt method prepare.

@Override
public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
    super.prepare(stormConf, context, collector);
    if (periodDurationMillis <= 0) {
        throw new IllegalArgumentException("expect 'profiler.period.duration' >= 0");
    }
    if (profileTimeToLiveMillis <= 0) {
        throw new IllegalArgumentException("expect 'profiler.ttl' >= 0");
    }
    if (profileTimeToLiveMillis < periodDurationMillis) {
        throw new IllegalArgumentException("expect 'profiler.ttl' >= 'profiler.period.duration'");
    }
    if (maxNumberOfRoutes <= 0) {
        throw new IllegalArgumentException("expect 'profiler.max.routes.per.bolt' > 0");
    }
    if (windowDurationMillis <= 0) {
        throw new IllegalArgumentException("expect 'profiler.window.duration' > 0");
    }
    if (windowDurationMillis > periodDurationMillis) {
        throw new IllegalArgumentException("expect 'profiler.period.duration' >= 'profiler.window.duration'");
    }
    if (periodDurationMillis % windowDurationMillis != 0) {
        throw new IllegalArgumentException("expect 'profiler.period.duration' % 'profiler.window.duration' == 0");
    }
    this.collector = collector;
    this.parser = new JSONParser();
    this.messageDistributor = new DefaultMessageDistributor(periodDurationMillis, profileTimeToLiveMillis, maxNumberOfRoutes);
    this.configurations = new ProfilerConfigurations();
    this.flushSignal = new FixedFrequencyFlushSignal(periodDurationMillis);
    setupZookeeper();
}

Example 2 with ProfilerConfigurations

use of org.apache.metron.common.configuration.profiler.ProfilerConfigurations in project metron by apache.

the class ProfileBuilderBoltTest method createBolt.

/**
 * Create a ProfileBuilderBolt to test.
 *
 * @param windowDuration The event window duration.
 * @param windowDurationUnits The units of the event window duration.
 * @return A {@link ProfileBuilderBolt} to test.
 */
private ProfileBuilderBolt createBolt(int windowDuration, TimeUnit windowDurationUnits) throws IOException {
    // defines the zk configurations accessible from the bolt
    ProfilerConfigurations configurations = new ProfilerConfigurations();
    configurations.updateGlobalConfig(Collections.emptyMap());
    emitter = new HBaseEmitter();
    ProfileBuilderBolt bolt = (ProfileBuilderBolt) new ProfileBuilderBolt().withProfileTimeToLive(30, TimeUnit.MINUTES).withMaxNumberOfRoutes(Long.MAX_VALUE).withZookeeperClient(client).withZookeeperCache(cache).withEmitter(emitter).withProfilerConfigurations(configurations).withPeriodDuration(1, TimeUnit.MINUTES).withTumblingWindow(new BaseWindowedBolt.Duration(windowDuration, windowDurationUnits));
    bolt.prepare(new HashMap<>(), topologyContext, outputCollector);
    // set the flush signal AFTER calling 'prepare'
    bolt.withFlushSignal(flushSignal);
    return bolt;
}

Example 3 with ProfilerConfigurations

use of org.apache.metron.common.configuration.profiler.ProfilerConfigurations in project metron by apache.

the class ZKConfigurationsCacheIntegrationTest method validateBaseWrite.

@Test
public void validateBaseWrite() throws Exception {
    File globalConfigFile = new File(TestConstants.SAMPLE_CONFIG_PATH + "/global.json");
    Map<String, Object> expectedGlobalConfig = JSONUtils.INSTANCE.load(globalConfigFile, JSONUtils.MAP_SUPPLIER);
    // indexing
    {
        File inFile = new File(TestConstants.SAMPLE_CONFIG_PATH + "/indexing/test.json");
        Map<String, Object> expectedConfig = JSONUtils.INSTANCE.load(inFile, JSONUtils.MAP_SUPPLIER);
        IndexingConfigurations config = cache.get(IndexingConfigurations.class);
        assertEventually(() -> Assert.assertEquals(expectedConfig, config.getSensorIndexingConfig("test")));
        assertEventually(() -> Assert.assertEquals(expectedGlobalConfig, config.getGlobalConfig()));
        assertEventually(() -> Assert.assertNull(config.getSensorIndexingConfig("notthere", false)));
    }
    // enrichment
    {
        File inFile = new File(TestConstants.SAMPLE_CONFIG_PATH + "/enrichments/test.json");
        SensorEnrichmentConfig expectedConfig = JSONUtils.INSTANCE.load(inFile, SensorEnrichmentConfig.class);
        EnrichmentConfigurations config = cache.get(EnrichmentConfigurations.class);
        assertEventually(() -> Assert.assertEquals(expectedConfig, config.getSensorEnrichmentConfig("test")));
        assertEventually(() -> Assert.assertEquals(expectedGlobalConfig, config.getGlobalConfig()));
        assertEventually(() -> Assert.assertNull(config.getSensorEnrichmentConfig("notthere")));
    }
    // parsers
    {
        File inFile = new File(TestConstants.PARSER_CONFIGS_PATH + "/parsers/bro.json");
        SensorParserConfig expectedConfig = JSONUtils.INSTANCE.load(inFile, SensorParserConfig.class);
        ParserConfigurations config = cache.get(ParserConfigurations.class);
        assertEventually(() -> Assert.assertEquals(expectedConfig, config.getSensorParserConfig("bro")));
        assertEventually(() -> Assert.assertEquals(expectedGlobalConfig, config.getGlobalConfig()));
        assertEventually(() -> Assert.assertNull(config.getSensorParserConfig("notthere")));
    }
    // profiler
    {
        File inFile = new File(profilerDir, "/readme-example-1/profiler.json");
        ProfilerConfig expectedConfig = JSONUtils.INSTANCE.load(inFile, ProfilerConfig.class);
        ProfilerConfigurations config = cache.get(ProfilerConfigurations.class);
        assertEventually(() -> Assert.assertEquals(expectedConfig, config.getProfilerConfig()));
        assertEventually(() -> Assert.assertEquals(expectedGlobalConfig, config.getGlobalConfig()));
    }
}

Example 4 with ProfilerConfigurations

use of org.apache.metron.common.configuration.profiler.ProfilerConfigurations in project metron by apache.

the class ProfileBuilderBolt method setupZookeeper.

private void setupZookeeper() {
    try {
        if (zookeeperClient == null) {
            RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3);
            zookeeperClient = CuratorFrameworkFactory.newClient(zookeeperUrl, retryPolicy);
        }
        zookeeperClient.start();
        // this is temporary to ensure that any validation passes. the individual bolt
        // will reinitialize stellar to dynamically pull from zookeeper.
        ConfigurationsUtils.setupStellarStatically(zookeeperClient);
        if (zookeeperCache == null) {
            ConfigurationsUpdater<ProfilerConfigurations> updater = createUpdater();
            SimpleEventListener listener = new SimpleEventListener.Builder().with(updater::update, TreeCacheEvent.Type.NODE_ADDED, TreeCacheEvent.Type.NODE_UPDATED).with(updater::delete, TreeCacheEvent.Type.NODE_REMOVED).build();
            zookeeperCache = new ZKCache.Builder().withClient(zookeeperClient).withListener(listener).withRoot(Constants.ZOOKEEPER_TOPOLOGY_ROOT).build();
            updater.forceUpdate(zookeeperClient);
            zookeeperCache.start();
        }
    } catch (Exception e) {
        LOG.error(e.getMessage(), e);
        throw new RuntimeException(e);
    }
}

Example 5 with ProfilerConfigurations

use of org.apache.metron.common.configuration.profiler.ProfilerConfigurations in project metron by apache.

the class ProfileBuilderBoltTest method testEmitters.

/**
 * A {@link ProfileMeasurement} is built for each profile/entity pair.  The measurement should be emitted to each
 * destination defined by the profile. By default, a profile uses both Kafka and HBase as destinations.
 */
@Test
public void testEmitters() throws Exception {
    // defines the zk configurations accessible from the bolt
    ProfilerConfigurations configurations = new ProfilerConfigurations();
    configurations.updateGlobalConfig(Collections.emptyMap());
    // create the bolt with 3 destinations
    ProfileBuilderBolt bolt = (ProfileBuilderBolt) new ProfileBuilderBolt().withProfileTimeToLive(30, TimeUnit.MINUTES).withPeriodDuration(10, TimeUnit.MINUTES).withMaxNumberOfRoutes(Long.MAX_VALUE).withZookeeperClient(client).withZookeeperCache(cache).withEmitter(new TestEmitter("destination1")).withEmitter(new TestEmitter("destination2")).withEmitter(new TestEmitter("destination3")).withProfilerConfigurations(configurations).withTumblingWindow(new BaseWindowedBolt.Duration(10, TimeUnit.MINUTES));
    bolt.prepare(new HashMap<>(), topologyContext, outputCollector);
    // signal the bolt to flush
    bolt.withFlushSignal(flushSignal);
    flushSignal.setFlushNow(true);
    // execute the bolt
    Tuple tuple1 = createTuple("entity", message1, profile1, System.currentTimeMillis());
    TupleWindow window = createWindow(tuple1);
    bolt.execute(window);
    // validate measurements emitted to each
    verify(outputCollector, times(1)).emit(eq("destination1"), any());
    verify(outputCollector, times(1)).emit(eq("destination2"), any());
    verify(outputCollector, times(1)).emit(eq("destination3"), any());
}