Examples with MemoryTransactionalSpout backtype.storm.testing.MemoryTransactionalSpout used on opensource projects

Example 1 with MemoryTransactionalSpout

use of backtype.storm.testing.MemoryTransactionalSpout in project storm-hbase by jrkinley.

the class HBaseCountersBatchTopology method main.

/**
   * @param args
   * @throws InterruptedException 
   */
public static void main(String[] args) throws InterruptedException {
    // Add transactional spout
    MemoryTransactionalSpout spout = new MemoryTransactionalSpout(values, new Fields("shortid", "url", "user", "date"), 3);
    TransactionalTopologyBuilder builder = new TransactionalTopologyBuilder("shorturl-count", "spout", spout, 2);
    // Build TupleTableConifg
    TupleTableConfig ttConfig = new TupleTableConfig("shorturl", "shortid");
    ttConfig.setBatch(false);
    ttConfig.addColumn("data", "clicks");
    ttConfig.addColumn("daily", "date");
    builder.setBolt("hbase-counters", new HBaseCountersBatchBolt(ttConfig), 2).fieldsGrouping("spout", new Fields("shortid"));
    LocalCluster cluster = new LocalCluster();
    Config stormConfig = new Config();
    stormConfig.setDebug(true);
    stormConfig.setMaxSpoutPending(3);
    cluster.submitTopology("hbase-example", stormConfig, builder.buildTopology());
    Thread.sleep(10000);
    cluster.shutdown();
}

Example 2 with MemoryTransactionalSpout

use of backtype.storm.testing.MemoryTransactionalSpout in project jstorm by alibaba.

the class TransactionalGlobalCount method test.

public static void test() {
    MemoryTransactionalSpout spout = new MemoryTransactionalSpout(DATA, new Fields("word"), PARTITION_TAKE_PER_BATCH);
    TransactionalTopologyBuilder builder = new TransactionalTopologyBuilder("global-count", "spout", spout, 1);
    builder.setBolt("partial-count", new BatchCount(), 2).noneGrouping("spout");
    builder.setBolt("sum", new UpdateGlobalCount(), 1).globalGrouping("partial-count");
    conf.setDebug(true);
    conf.setMaxSpoutPending(3);
    String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\.");
    String topologyName = className[className.length - 1];
    try {
        JStormHelper.runTopology(builder.buildTopology(), topologyName, conf, 60, new JStormHelper.CheckAckedFail(conf), isLocal);
    } catch (Exception e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
        Assert.fail("Failed");
    }
}

Example 3 with MemoryTransactionalSpout

use of backtype.storm.testing.MemoryTransactionalSpout in project jstorm by alibaba.

the class TransactionalWords method test.

public static void test() {
    MemoryTransactionalSpout spout = new MemoryTransactionalSpout(DATA, new Fields("word"), PARTITION_TAKE_PER_BATCH);
    TransactionalTopologyBuilder builder = new TransactionalTopologyBuilder("top-n-words", "spout", spout, 2);
    builder.setBolt("count", new KeyedCountUpdater(), 5).fieldsGrouping("spout", new Fields("word"));
    builder.setBolt("bucketize", new Bucketize()).noneGrouping("count");
    builder.setBolt("buckets", new BucketCountUpdater(), 5).fieldsGrouping("bucketize", new Fields("bucket"));
    String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\.");
    String topologyName = className[className.length - 1];
    try {
        JStormHelper.runTopology(builder.buildTopology(), topologyName, conf, 60, new JStormHelper.CheckAckedFail(conf), isLocal);
    } catch (Exception e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
        Assert.fail("Failed");
    }
}

Example 4 with MemoryTransactionalSpout

use of backtype.storm.testing.MemoryTransactionalSpout in project jstorm by alibaba.

the class TransactionalWordsTest method test_transaction_word.

@Test
public void test_transaction_word() {
    try {
        MemoryTransactionalSpout spout = new MemoryTransactionalSpout(DATA, new Fields("word"), PARTITION_TAKE_PER_BATCH);
        TransactionalTopologyBuilder builder = new TransactionalTopologyBuilder("top-n-words", "spout", spout, 2);
        builder.setBolt("count", new KeyedCountUpdater(), 5).fieldsGrouping("spout", new Fields("word"));
        builder.setBolt("bucketize", new Bucketize()).shuffleGrouping("count");
        builder.setBolt("buckets", new BucketCountUpdater(), 5).fieldsGrouping("bucketize", new Fields("bucket"));
        LocalCluster cluster = new LocalCluster();
        Config config = new Config();
        config.setDebug(true);
        config.setMaxSpoutPending(3);
        cluster.submitTopology("top-n-topology", config, builder.buildTopology());
        JStormUtils.sleepMs(60 * 1000);
        assertEquals(false, outOfOrder.get());
        assertNotSame(0, receiveCounter1.get());
        assertNotSame(0, receiveCounter2.get());
    //cluster.killTopology("top-n-topology");
    //cluster.shutdown();
    } catch (Exception e) {
        e.printStackTrace();
        Assert.fail("Failed to run simple transaction");
    }
}