Search in sources :

Example 1 with StateFactory

use of storm.trident.state.StateFactory in project jstorm by alibaba.

the class Stream method window.

private Stream window(WindowConfig windowConfig, WindowsStoreFactory windowStoreFactory, Fields inputFields, Aggregator aggregator, Fields functionFields, boolean storeTuplesInStore) {
    projectionValidation(inputFields);
    windowConfig.validate();
    Fields fields = addTriggerField(functionFields);
    // when storeTuplesInStore is false then the given windowStoreFactory is only used to store triggers and
    // that store is passed to WindowStateUpdater to remove them after committing the batch.
    Stream stream = _topology.addSourcedNode(this, new ProcessorNode(_topology.getUniqueStreamId(), _name, fields, fields, new WindowTridentProcessor(windowConfig, _topology.getUniqueWindowId(), windowStoreFactory, inputFields, aggregator, storeTuplesInStore)));
    Stream effectiveStream = stream.project(functionFields);
    // create StateUpdater with the given windowStoreFactory to remove triggered aggregation results form store
    // when they are successfully processed.
    StateFactory stateFactory = new WindowsStateFactory();
    StateUpdater stateUpdater = new WindowsStateUpdater(windowStoreFactory);
    stream.partitionPersist(stateFactory, new Fields(WindowTridentProcessor.TRIGGER_FIELD_NAME), stateUpdater, new Fields());
    return effectiveStream;
}
Also used : Fields(backtype.storm.tuple.Fields) ProcessorNode(storm.trident.planner.ProcessorNode) WindowsStateFactory(storm.trident.windowing.WindowsStateFactory) StateFactory(storm.trident.state.StateFactory) WindowsStateUpdater(storm.trident.windowing.WindowsStateUpdater) GroupedStream(storm.trident.fluent.GroupedStream) IAggregatableStream(storm.trident.fluent.IAggregatableStream) WindowTridentProcessor(storm.trident.windowing.WindowTridentProcessor) WindowsStateFactory(storm.trident.windowing.WindowsStateFactory) ReducerAggStateUpdater(storm.trident.operation.impl.ReducerAggStateUpdater) WindowsStateUpdater(storm.trident.windowing.WindowsStateUpdater) StateUpdater(storm.trident.state.StateUpdater) CombinerAggStateUpdater(storm.trident.operation.impl.CombinerAggStateUpdater)

Example 2 with StateFactory

use of storm.trident.state.StateFactory in project storm-hbase by jrkinley.

the class HBaseTridentAggregateTopology method main.

/**
   * @param args
   * @throws InterruptedException
   */
@SuppressWarnings({ "unchecked", "rawtypes" })
public static void main(String[] args) throws InterruptedException {
    List<Object> v0 = HBaseCountersBatchTopology.values.get(0).get(0);
    List<Object> v1 = HBaseCountersBatchTopology.values.get(0).get(1);
    List<Object> v2 = HBaseCountersBatchTopology.values.get(0).get(2);
    List<Object> v3 = HBaseCountersBatchTopology.values.get(0).get(3);
    List<Object> v4 = HBaseCountersBatchTopology.values.get(0).get(4);
    List<Object> v5 = HBaseCountersBatchTopology.values.get(1).get(0);
    List<Object> v6 = HBaseCountersBatchTopology.values.get(1).get(1);
    List<Object> v7 = HBaseCountersBatchTopology.values.get(1).get(2);
    List<Object> v8 = HBaseCountersBatchTopology.values.get(2).get(0);
    List<Object> v9 = HBaseCountersBatchTopology.values.get(2).get(1);
    List<Object> v10 = HBaseCountersBatchTopology.values.get(2).get(2);
    HBaseCountersBatchTopology.values.values();
    FixedBatchSpout spout = new FixedBatchSpout(new Fields("shortid", "url", "user", "date"), 3, v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10);
    spout.setCycle(false);
    TridentConfig config = new TridentConfig("shorturl", "shortid");
    config.setBatch(false);
    StateFactory state = HBaseAggregateState.transactional(config);
    TridentTopology topology = new TridentTopology();
    topology.newStream("spout", spout).each(new Fields("shortid", "date"), new DatePartitionFunction(), new Fields("cf", "cq")).project(new Fields("shortid", "cf", "cq")).groupBy(new Fields("shortid", "cf", "cq")).persistentAggregate(state, new Count(), new Fields("count"));
    Config conf = new Config();
    LocalCluster cluster = new LocalCluster();
    cluster.submitTopology("hbase-trident-aggregate", conf, topology.build());
    Utils.sleep(5000);
    cluster.shutdown();
}
Also used : FixedBatchSpout(storm.trident.testing.FixedBatchSpout) LocalCluster(backtype.storm.LocalCluster) Fields(backtype.storm.tuple.Fields) StateFactory(storm.trident.state.StateFactory) TridentTopology(storm.trident.TridentTopology) Config(backtype.storm.Config) TridentConfig(backtype.storm.contrib.hbase.utils.TridentConfig) TridentConfig(backtype.storm.contrib.hbase.utils.TridentConfig) Count(storm.trident.operation.builtin.Count)

Aggregations

Fields (backtype.storm.tuple.Fields)2 StateFactory (storm.trident.state.StateFactory)2 Config (backtype.storm.Config)1 LocalCluster (backtype.storm.LocalCluster)1 TridentConfig (backtype.storm.contrib.hbase.utils.TridentConfig)1 TridentTopology (storm.trident.TridentTopology)1 GroupedStream (storm.trident.fluent.GroupedStream)1 IAggregatableStream (storm.trident.fluent.IAggregatableStream)1 Count (storm.trident.operation.builtin.Count)1 CombinerAggStateUpdater (storm.trident.operation.impl.CombinerAggStateUpdater)1 ReducerAggStateUpdater (storm.trident.operation.impl.ReducerAggStateUpdater)1 ProcessorNode (storm.trident.planner.ProcessorNode)1 StateUpdater (storm.trident.state.StateUpdater)1 FixedBatchSpout (storm.trident.testing.FixedBatchSpout)1 WindowTridentProcessor (storm.trident.windowing.WindowTridentProcessor)1 WindowsStateFactory (storm.trident.windowing.WindowsStateFactory)1 WindowsStateUpdater (storm.trident.windowing.WindowsStateUpdater)1