use of backtype.storm.transactional.TransactionalTopologyBuilder in project storm-hbase by jrkinley.
the class HBaseCountersBatchTopology method main.
/**
* @param args
* @throws InterruptedException
*/
public static void main(String[] args) throws InterruptedException {
// Add transactional spout
MemoryTransactionalSpout spout = new MemoryTransactionalSpout(values, new Fields("shortid", "url", "user", "date"), 3);
TransactionalTopologyBuilder builder = new TransactionalTopologyBuilder("shorturl-count", "spout", spout, 2);
// Build TupleTableConifg
TupleTableConfig ttConfig = new TupleTableConfig("shorturl", "shortid");
ttConfig.setBatch(false);
ttConfig.addColumn("data", "clicks");
ttConfig.addColumn("daily", "date");
builder.setBolt("hbase-counters", new HBaseCountersBatchBolt(ttConfig), 2).fieldsGrouping("spout", new Fields("shortid"));
LocalCluster cluster = new LocalCluster();
Config stormConfig = new Config();
stormConfig.setDebug(true);
stormConfig.setMaxSpoutPending(3);
cluster.submitTopology("hbase-example", stormConfig, builder.buildTopology());
Thread.sleep(10000);
cluster.shutdown();
}
use of backtype.storm.transactional.TransactionalTopologyBuilder in project jstorm by alibaba.
the class TransactionalGlobalCount method test.
public static void test() {
MemoryTransactionalSpout spout = new MemoryTransactionalSpout(DATA, new Fields("word"), PARTITION_TAKE_PER_BATCH);
TransactionalTopologyBuilder builder = new TransactionalTopologyBuilder("global-count", "spout", spout, 1);
builder.setBolt("partial-count", new BatchCount(), 2).noneGrouping("spout");
builder.setBolt("sum", new UpdateGlobalCount(), 1).globalGrouping("partial-count");
conf.setDebug(true);
conf.setMaxSpoutPending(3);
String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\.");
String topologyName = className[className.length - 1];
try {
JStormHelper.runTopology(builder.buildTopology(), topologyName, conf, 60, new JStormHelper.CheckAckedFail(conf), isLocal);
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
Assert.fail("Failed");
}
}
use of backtype.storm.transactional.TransactionalTopologyBuilder in project jstorm by alibaba.
the class TransactionalWords method test.
public static void test() {
MemoryTransactionalSpout spout = new MemoryTransactionalSpout(DATA, new Fields("word"), PARTITION_TAKE_PER_BATCH);
TransactionalTopologyBuilder builder = new TransactionalTopologyBuilder("top-n-words", "spout", spout, 2);
builder.setBolt("count", new KeyedCountUpdater(), 5).fieldsGrouping("spout", new Fields("word"));
builder.setBolt("bucketize", new Bucketize()).noneGrouping("count");
builder.setBolt("buckets", new BucketCountUpdater(), 5).fieldsGrouping("bucketize", new Fields("bucket"));
String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\.");
String topologyName = className[className.length - 1];
try {
JStormHelper.runTopology(builder.buildTopology(), topologyName, conf, 60, new JStormHelper.CheckAckedFail(conf), isLocal);
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
Assert.fail("Failed");
}
}
use of backtype.storm.transactional.TransactionalTopologyBuilder in project jstorm by alibaba.
the class TransactionalWordsTest method test_transaction_word.
@Test
public void test_transaction_word() {
try {
MemoryTransactionalSpout spout = new MemoryTransactionalSpout(DATA, new Fields("word"), PARTITION_TAKE_PER_BATCH);
TransactionalTopologyBuilder builder = new TransactionalTopologyBuilder("top-n-words", "spout", spout, 2);
builder.setBolt("count", new KeyedCountUpdater(), 5).fieldsGrouping("spout", new Fields("word"));
builder.setBolt("bucketize", new Bucketize()).shuffleGrouping("count");
builder.setBolt("buckets", new BucketCountUpdater(), 5).fieldsGrouping("bucketize", new Fields("bucket"));
LocalCluster cluster = new LocalCluster();
Config config = new Config();
config.setDebug(true);
config.setMaxSpoutPending(3);
cluster.submitTopology("top-n-topology", config, builder.buildTopology());
JStormUtils.sleepMs(60 * 1000);
assertEquals(false, outOfOrder.get());
assertNotSame(0, receiveCounter1.get());
assertNotSame(0, receiveCounter2.get());
// cluster.killTopology("top-n-topology");
// cluster.shutdown();
} catch (Exception e) {
e.printStackTrace();
Assert.fail("Failed to run simple transaction");
}
}
Aggregations