Examples with ValueOnlyColumnBuilder org.apache.metron.profiler.hbase.ValueOnlyColumnBuilder used on opensource projects

Example 1 with ValueOnlyColumnBuilder

use of org.apache.metron.profiler.hbase.ValueOnlyColumnBuilder in project metron by apache.

the class HBaseProfilerClientTest method setup.

@BeforeEach
public void setup() {
    provider = new MockHBaseTableProvider();
    executor = new DefaultStellarStatefulExecutor();
    MockHBaseTableProvider.addToCache(tableName, columnFamily);
    // writes values to be read during testing
    long periodDurationMillis = periodUnits.toMillis(periodDuration);
    RowKeyBuilder rowKeyBuilder = new SaltyRowKeyBuilder();
    ColumnBuilder columnBuilder = new ValueOnlyColumnBuilder(columnFamily);
    profileWriter = new ProfileWriter(rowKeyBuilder, columnBuilder, provider, periodDurationMillis, tableName, null);
    client = new HBaseProfilerClient(provider, rowKeyBuilder, columnBuilder, periodDurationMillis, tableName, null);
}

Example 2 with ValueOnlyColumnBuilder

use of org.apache.metron.profiler.hbase.ValueOnlyColumnBuilder in project metron by apache.

the class ProfilerIntegrationTest method setupBeforeClass.

@BeforeClass
public static void setupBeforeClass() throws UnableToStartException {
    columnBuilder = new ValueOnlyColumnBuilder(columnFamily);
    // storm topology properties
    final Properties topologyProperties = new Properties() {

        {
            // storm settings
            setProperty("profiler.workers", "1");
            setProperty("profiler.executors", "0");
            setProperty("storm.auto.credentials", "[]");
            setProperty("topology.auto-credentials", "[]");
            setProperty("topology.message.timeout.secs", "60");
            setProperty("topology.max.spout.pending", "100000");
            // kafka settings
            setProperty("profiler.input.topic", inputTopic);
            setProperty("profiler.output.topic", outputTopic);
            setProperty("kafka.start", "UNCOMMITTED_EARLIEST");
            setProperty("kafka.security.protocol", "PLAINTEXT");
            // hbase settings
            setProperty("profiler.hbase.salt.divisor", Integer.toString(saltDivisor));
            setProperty("profiler.hbase.table", tableName);
            setProperty("profiler.hbase.column.family", columnFamily);
            setProperty("profiler.hbase.batch", "10");
            setProperty("profiler.hbase.flush.interval.seconds", "1");
            setProperty("hbase.provider.impl", "" + MockHBaseTableProvider.class.getName());
            // profile settings
            setProperty("profiler.period.duration", Long.toString(periodDurationMillis));
            setProperty("profiler.period.duration.units", "MILLISECONDS");
            setProperty("profiler.ttl", Long.toString(profileTimeToLiveMillis));
            setProperty("profiler.ttl.units", "MILLISECONDS");
            setProperty("profiler.window.duration", Long.toString(windowDurationMillis));
            setProperty("profiler.window.duration.units", "MILLISECONDS");
            setProperty("profiler.window.lag", Long.toString(windowLagMillis));
            setProperty("profiler.window.lag.units", "MILLISECONDS");
            setProperty("profiler.max.routes.per.bolt", Long.toString(maxRoutesPerBolt));
        }
    };
    // create the mock table
    profilerTable = (MockHTable) MockHBaseTableProvider.addToCache(tableName, columnFamily);
    zkComponent = getZKServerComponent(topologyProperties);
    // create the input and output topics
    kafkaComponent = getKafkaComponent(topologyProperties, Arrays.asList(new KafkaComponent.Topic(inputTopic, 1), new KafkaComponent.Topic(outputTopic, 1)));
    // upload profiler configuration to zookeeper
    configUploadComponent = new ConfigUploadComponent().withTopologyProperties(topologyProperties);
    // load flux definition for the profiler topology
    fluxComponent = new FluxTopologyComponent.Builder().withTopologyLocation(new File(FLUX_PATH)).withTopologyName("profiler").withTopologyProperties(topologyProperties).build();
    // start all components
    runner = new ComponentRunner.Builder().withComponent("zk", zkComponent).withComponent("kafka", kafkaComponent).withComponent("config", configUploadComponent).withComponent("storm", fluxComponent).withMillisecondsBetweenAttempts(15000).withNumRetries(10).withCustomShutdownOrder(new String[] { "storm", "config", "kafka", "zk" }).build();
    runner.start();
}

Example 3 with ValueOnlyColumnBuilder

use of org.apache.metron.profiler.hbase.ValueOnlyColumnBuilder in project metron by apache.

the class GetProfile method getColumnBuilder.

/**
 * Creates the ColumnBuilder to use in accessing the profile data.
 * @param global The global configuration.
 */
private ColumnBuilder getColumnBuilder(Map<String, Object> global) {
    ColumnBuilder columnBuilder;
    String columnFamily = PROFILER_COLUMN_FAMILY.get(global, String.class);
    columnBuilder = new ValueOnlyColumnBuilder(columnFamily);
    return columnBuilder;
}

Example 4 with ValueOnlyColumnBuilder

use of org.apache.metron.profiler.hbase.ValueOnlyColumnBuilder in project metron by apache.

the class GetProfileTest method setup.

/**
 * This method sets up the configuration context for both writing profile data
 * (using profileWriter to mock the complex process of what the Profiler topology
 * actually does), and then reading that profile data (thereby testing the PROFILE_GET
 * Stellar client implemented in GetProfile).
 *
 * It runs at @BeforeEach time, and sets testclass global variables used by the writers and readers.
 * The various writers and readers are in each test case, not here.
 *
 * @return void
 */
@BeforeEach
public void setup() {
    state = new HashMap<>();
    final Table table = MockHBaseTableProvider.addToCache(tableName, columnFamily);
    // used to write values to be read during testing
    long periodDurationMillis = TimeUnit.MINUTES.toMillis(15);
    RowKeyBuilder rowKeyBuilder = new SaltyRowKeyBuilder();
    ColumnBuilder columnBuilder = new ValueOnlyColumnBuilder(columnFamily);
    profileWriter = new ProfileWriter(rowKeyBuilder, columnBuilder, new MockHBaseTableProvider(), periodDurationMillis, tableName, null);
    // global properties
    Map<String, Object> global = new HashMap<String, Object>() {

        {
            put(PROFILER_HBASE_TABLE.getKey(), tableName);
            put(PROFILER_COLUMN_FAMILY.getKey(), columnFamily);
            put(PROFILER_HBASE_TABLE_PROVIDER.getKey(), MockHBaseTableProvider.class.getName());
            put(PROFILER_PERIOD.getKey(), Long.toString(periodDuration));
            put(PROFILER_PERIOD_UNITS.getKey(), periodUnits.toString());
            put(PROFILER_SALT_DIVISOR.getKey(), Integer.toString(saltDivisor));
        }
    };
    // create the stellar execution environment
    executor = new DefaultStellarStatefulExecutor(new SimpleFunctionResolver().withClass(GetProfile.class).withClass(FixedLookback.class), new Context.Builder().with(Context.Capabilities.GLOBAL_CONFIG, () -> global).build());
}

Example 5 with ValueOnlyColumnBuilder

use of org.apache.metron.profiler.hbase.ValueOnlyColumnBuilder in project metron by apache.

the class VerboseProfileTest method setup.

@BeforeEach
public void setup() {
    state = new HashMap<>();
    final Table table = MockHBaseTableProvider.addToCache(tableName, columnFamily);
    TableProvider provider = new MockHBaseTableProvider();
    // used to write values to be read during testing
    long periodDurationMillis = TimeUnit.MINUTES.toMillis(15);
    RowKeyBuilder rowKeyBuilder = new SaltyRowKeyBuilder();
    ColumnBuilder columnBuilder = new ValueOnlyColumnBuilder(columnFamily);
    profileWriter = new ProfileWriter(rowKeyBuilder, columnBuilder, provider, periodDurationMillis, tableName, null);
    // global properties
    globals = new HashMap<String, Object>() {

        {
            put(PROFILER_HBASE_TABLE.getKey(), tableName);
            put(PROFILER_COLUMN_FAMILY.getKey(), columnFamily);
            put(PROFILER_HBASE_TABLE_PROVIDER.getKey(), MockHBaseTableProvider.class.getName());
            put(PROFILER_PERIOD.getKey(), Long.toString(periodDuration));
            put(PROFILER_PERIOD_UNITS.getKey(), periodUnits.toString());
            put(PROFILER_SALT_DIVISOR.getKey(), Integer.toString(saltDivisor));
        }
    };
    // create the stellar execution environment
    executor = new DefaultStellarStatefulExecutor(new SimpleFunctionResolver().withClass(VerboseProfile.class).withClass(FixedLookback.class), new Context.Builder().with(Context.Capabilities.GLOBAL_CONFIG, () -> globals).build());
}