Examples with StateFactory org.apache.storm.trident.state.StateFactory used on opensource projects

Example 6 with StateFactory

use of org.apache.storm.trident.state.StateFactory in project storm by apache.

the class TridentFileTopology method buildTopology.

public static StormTopology buildTopology(String hdfsUrl) {
    FixedBatchSpout spout = new FixedBatchSpout(new Fields("sentence", "key"), 1000, new Values("the cow jumped over the moon", 1l), new Values("the man went to the store and bought some candy", 2l), new Values("four score and seven years ago", 3l), new Values("how many apples can you eat", 4l), new Values("to be or not to be the person", 5l));
    spout.setCycle(true);
    TridentTopology topology = new TridentTopology();
    Stream stream = topology.newStream("spout1", spout);
    Fields hdfsFields = new Fields("sentence", "key");
    FileNameFormat fileNameFormat = new DefaultFileNameFormat().withPath("/tmp/trident").withPrefix("trident").withExtension(".txt");
    RecordFormat recordFormat = new DelimitedRecordFormat().withFields(hdfsFields);
    FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(5.0f, FileSizeRotationPolicy.Units.MB);
    HdfsState.Options options = new HdfsState.HdfsFileOptions().withFileNameFormat(fileNameFormat).withRecordFormat(recordFormat).withRotationPolicy(rotationPolicy).withFsUrl(hdfsUrl).withConfigKey("hdfs.config");
    StateFactory factory = new HdfsStateFactory().withOptions(options);
    TridentState state = stream.partitionPersist(factory, hdfsFields, new HdfsUpdater(), new Fields());
    return topology.build();
}

Example 7 with StateFactory

use of org.apache.storm.trident.state.StateFactory in project storm by apache.

the class Stream method window.

private Stream window(WindowConfig windowConfig, WindowsStoreFactory windowStoreFactory, Fields inputFields, Aggregator aggregator, Fields functionFields, boolean storeTuplesInStore) {
    projectionValidation(inputFields);
    windowConfig.validate();
    Fields fields = addTriggerField(functionFields);
    // when storeTuplesInStore is false then the given windowStoreFactory is only used to store triggers and
    // that store is passed to WindowStateUpdater to remove them after committing the batch.
    Stream stream = _topology.addSourcedNode(this, new ProcessorNode(_topology.getUniqueStreamId(), _name, fields, fields, new WindowTridentProcessor(windowConfig, _topology.getUniqueWindowId(), windowStoreFactory, inputFields, aggregator, storeTuplesInStore)));
    Stream effectiveStream = stream.project(functionFields);
    // create StateUpdater with the given windowStoreFactory to remove triggered aggregation results form store
    // when they are successfully processed.
    StateFactory stateFactory = new WindowsStateFactory();
    StateUpdater stateUpdater = new WindowsStateUpdater(windowStoreFactory);
    stream.partitionPersist(stateFactory, new Fields(WindowTridentProcessor.TRIGGER_FIELD_NAME), stateUpdater, new Fields());
    return effectiveStream;
}

Example 8 with StateFactory

use of org.apache.storm.trident.state.StateFactory in project storm by apache.

the class TridentSequenceTopology method buildTopology.

public static StormTopology buildTopology(String hdfsUrl) {
    FixedBatchSpout spout = new FixedBatchSpout(new Fields("sentence", "key"), 1000, new Values("the cow jumped over the moon", 1l), new Values("the man went to the store and bought some candy", 2l), new Values("four score and seven years ago", 3l), new Values("how many apples can you eat", 4l), new Values("to be or not to be the person", 5l));
    spout.setCycle(true);
    TridentTopology topology = new TridentTopology();
    Stream stream = topology.newStream("spout1", spout);
    Fields hdfsFields = new Fields("sentence", "key");
    FileNameFormat fileNameFormat = new DefaultFileNameFormat().withPath("/tmp/trident").withPrefix("trident").withExtension(".seq");
    FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(5.0f, FileSizeRotationPolicy.Units.MB);
    HdfsState.Options seqOpts = new HdfsState.SequenceFileOptions().withFileNameFormat(fileNameFormat).withSequenceFormat(new DefaultSequenceFormat("key", "sentence")).withRotationPolicy(rotationPolicy).withFsUrl(hdfsUrl).withConfigKey("hdfs.config").addRotationAction(new MoveFileAction().toDestination("/tmp/dest2/"));
    StateFactory factory = new HdfsStateFactory().withOptions(seqOpts);
    TridentState state = stream.partitionPersist(factory, hdfsFields, new HdfsUpdater(), new Fields());
    return topology.build();
}

Example 9 with StateFactory

use of org.apache.storm.trident.state.StateFactory in project storm by apache.

the class TridentHiveTopology method buildTopology.

public static StormTopology buildTopology(String metaStoreURI, String dbName, String tblName, Object keytab, Object principal) {
    int batchSize = 100;
    FixedBatchSpout spout = new FixedBatchSpout(batchSize);
    spout.setCycle(true);
    TridentTopology topology = new TridentTopology();
    Stream stream = topology.newStream("hiveTridentspout1", spout);
    String[] partNames = { "city", "state" };
    String[] colNames = { "id", "name", "phone", "street" };
    Fields hiveFields = new Fields("id", "name", "phone", "street", "city", "state");
    DelimitedRecordHiveMapper mapper = new DelimitedRecordHiveMapper().withColumnFields(new Fields(colNames)).withPartitionFields(new Fields(partNames));
    HiveOptions hiveOptions;
    if (keytab != null && principal != null) {
        hiveOptions = new HiveOptions(metaStoreURI, dbName, tblName, mapper).withTxnsPerBatch(10).withBatchSize(batchSize).withIdleTimeout(10).withCallTimeout(30000).withKerberosKeytab((String) keytab).withKerberosPrincipal((String) principal);
    } else {
        hiveOptions = new HiveOptions(metaStoreURI, dbName, tblName, mapper).withTxnsPerBatch(10).withBatchSize(batchSize).withCallTimeout(30000).withIdleTimeout(10);
    }
    StateFactory factory = new HiveStateFactory().withOptions(hiveOptions);
    TridentState state = stream.partitionPersist(factory, hiveFields, new HiveUpdater(), new Fields());
    return topology.build();
}

Example 10 with StateFactory

use of org.apache.storm.trident.state.StateFactory in project storm by apache.

the class WordCountTrident method buildTopology.

public static StormTopology buildTopology(String hbaseRoot) {
    Fields fields = new Fields("word", "count");
    FixedBatchSpout spout = new FixedBatchSpout(fields, 4, new Values("storm", 1), new Values("trident", 1), new Values("needs", 1), new Values("javadoc", 1));
    spout.setCycle(true);
    TridentHBaseMapper tridentHBaseMapper = new SimpleTridentHBaseMapper().withColumnFamily("cf").withColumnFields(new Fields("word")).withCounterFields(new Fields("count")).withRowKeyField("word");
    HBaseValueMapper rowToStormValueMapper = new WordCountValueMapper();
    HBaseProjectionCriteria projectionCriteria = new HBaseProjectionCriteria();
    projectionCriteria.addColumn(new HBaseProjectionCriteria.ColumnMetaData("cf", "count"));
    HBaseState.Options options = new HBaseState.Options().withConfigKey(hbaseRoot).withDurability(Durability.SYNC_WAL).withMapper(tridentHBaseMapper).withProjectionCriteria(projectionCriteria).withRowToStormValueMapper(rowToStormValueMapper).withTableName("WordCount");
    StateFactory factory = new HBaseStateFactory(options);
    TridentTopology topology = new TridentTopology();
    Stream stream = topology.newStream("spout1", spout);
    stream.partitionPersist(factory, fields, new HBaseUpdater(), new Fields());
    TridentState state = topology.newStaticState(factory);
    stream = stream.stateQuery(state, new Fields("word"), new HBaseQuery(), new Fields("columnName", "columnValue"));
    stream.each(new Fields("word", "columnValue"), new PrintFunction(), new Fields());
    return topology.build();
}