Examples with Fields org.apache.storm.tuple.Fields used on opensource projects

Example 41 with Fields

use of org.apache.storm.tuple.Fields in project storm by apache.

the class WordCountTrident method buildTopology.

public static StormTopology buildTopology(String url, String collectionName) {
    Fields fields = new Fields("word", "count");
    FixedBatchSpout spout = new FixedBatchSpout(fields, 4, new Values("storm", 1), new Values("trident", 1), new Values("needs", 1), new Values("javadoc", 1));
    spout.setCycle(true);
    MongoMapper mapper = new SimpleMongoMapper().withFields("word", "count");
    MongoState.Options options = new MongoState.Options().withUrl(url).withCollectionName(collectionName).withMapper(mapper);
    StateFactory factory = new MongoStateFactory(options);
    TridentTopology topology = new TridentTopology();
    Stream stream = topology.newStream("spout1", spout);
    stream.partitionPersist(factory, fields, new MongoStateUpdater(), new Fields());
    TridentState state = topology.newStaticState(factory);
    stream = stream.stateQuery(state, new Fields("word"), new MongoStateQuery(), new Fields("columnName", "columnValue"));
    stream.each(new Fields("word", "columnValue"), new PrintFunction(), new Fields());
    return topology.build();
}

Example 42 with Fields

use of org.apache.storm.tuple.Fields in project storm by apache.

the class WordCountTridentMap method buildTopology.

public static StormTopology buildTopology(String url, String collectionName) {
    Fields fields = new Fields("word", "count");
    FixedBatchSpout spout = new FixedBatchSpout(fields, 4, new Values("storm", 1), new Values("trident", 1), new Values("needs", 1), new Values("javadoc", 1));
    spout.setCycle(true);
    MongoMapper mapper = new SimpleMongoMapper().withFields("word", "count");
    QueryFilterCreator filterCreator = new SimpleQueryFilterCreator().withField("word");
    MongoMapState.Options options = new MongoMapState.Options();
    options.url = url;
    options.collectionName = collectionName;
    options.mapper = mapper;
    options.queryCreator = filterCreator;
    StateFactory factory = MongoMapState.transactional(options);
    TridentTopology topology = new TridentTopology();
    Stream stream = topology.newStream("spout1", spout);
    TridentState state = stream.groupBy(new Fields("word")).persistentAggregate(factory, new Fields("count"), new Sum(), new Fields("sum"));
    stream.stateQuery(state, new Fields("word"), new MapGet(), new Fields("sum")).each(new Fields("word", "sum"), new PrintFunction(), new Fields());
    return topology.build();
}

Example 43 with Fields

use of org.apache.storm.tuple.Fields in project storm by apache.

the class TridentEsTopology method main.

public static void main(String[] args) throws Exception {
    int batchSize = 100;
    FixedBatchSpout spout = new FixedBatchSpout(batchSize);
    spout.setCycle(true);
    TridentTopology topology = new TridentTopology();
    Stream stream = topology.newStream("spout", spout);
    EsConfig esConfig = new EsConfig(EsConstants.clusterName, new String[] { "localhost:9300" });
    Fields esFields = new Fields("index", "type", "source");
    EsTupleMapper tupleMapper = EsTestUtil.generateDefaultTupleMapper();
    StateFactory factory = new EsStateFactory(esConfig, tupleMapper);
    TridentState state = stream.partitionPersist(factory, esFields, new EsUpdater(), new Fields());
    EsTestUtil.startEsNode();
    EsTestUtil.waitForSeconds(5);
    try (LocalCluster cluster = new LocalCluster();
        LocalTopology topo = cluster.submitTopology(TOPOLOGY_NAME, null, topology.build())) {
        EsTestUtil.waitForSeconds(20);
    }
    System.exit(0);
}

Example 44 with Fields

use of org.apache.storm.tuple.Fields in project storm by apache.

the class TridentFileTopology method buildTopology.

public static StormTopology buildTopology(String hdfsUrl) {
    FixedBatchSpout spout = new FixedBatchSpout(new Fields("sentence", "key"), 1000, new Values("the cow jumped over the moon", 1l), new Values("the man went to the store and bought some candy", 2l), new Values("four score and seven years ago", 3l), new Values("how many apples can you eat", 4l), new Values("to be or not to be the person", 5l));
    spout.setCycle(true);
    TridentTopology topology = new TridentTopology();
    Stream stream = topology.newStream("spout1", spout);
    Fields hdfsFields = new Fields("sentence", "key");
    FileNameFormat fileNameFormat = new DefaultFileNameFormat().withPath("/tmp/trident").withPrefix("trident").withExtension(".txt");
    RecordFormat recordFormat = new DelimitedRecordFormat().withFields(hdfsFields);
    FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(5.0f, FileSizeRotationPolicy.Units.MB);
    HdfsState.Options options = new HdfsState.HdfsFileOptions().withFileNameFormat(fileNameFormat).withRecordFormat(recordFormat).withRotationPolicy(rotationPolicy).withFsUrl(hdfsUrl).withConfigKey("hdfs.config");
    StateFactory factory = new HdfsStateFactory().withOptions(options);
    TridentState state = stream.partitionPersist(factory, hdfsFields, new HdfsUpdater(), new Fields());
    return topology.build();
}

Example 45 with Fields

use of org.apache.storm.tuple.Fields in project storm by apache.

the class ThroughputVsLatency method main.

public static void main(String[] args) throws Exception {
    long ratePerSecond = 500;
    if (args != null && args.length > 0) {
        ratePerSecond = Long.valueOf(args[0]);
    }
    int parallelism = 4;
    if (args != null && args.length > 1) {
        parallelism = Integer.valueOf(args[1]);
    }
    int numMins = 5;
    if (args != null && args.length > 2) {
        numMins = Integer.valueOf(args[2]);
    }
    String name = "wc-test";
    if (args != null && args.length > 3) {
        name = args[3];
    }
    Config conf = new Config();
    HttpForwardingMetricsServer metricServer = new HttpForwardingMetricsServer(conf) {

        @Override
        public void handle(TaskInfo taskInfo, Collection<DataPoint> dataPoints) {
            String worker = taskInfo.srcWorkerHost + ":" + taskInfo.srcWorkerPort;
            for (DataPoint dp : dataPoints) {
                if ("comp-lat-histo".equals(dp.name) && dp.value instanceof Histogram) {
                    synchronized (_histo) {
                        _histo.add((Histogram) dp.value);
                    }
                } else if ("CPU".equals(dp.name) && dp.value instanceof Map) {
                    Map<Object, Object> m = (Map<Object, Object>) dp.value;
                    Object sys = m.get("sys-ms");
                    if (sys instanceof Number) {
                        _systemCPU.getAndAdd(((Number) sys).longValue());
                    }
                    Object user = m.get("user-ms");
                    if (user instanceof Number) {
                        _userCPU.getAndAdd(((Number) user).longValue());
                    }
                } else if (dp.name.startsWith("GC/") && dp.value instanceof Map) {
                    Map<Object, Object> m = (Map<Object, Object>) dp.value;
                    Object count = m.get("count");
                    if (count instanceof Number) {
                        _gcCount.getAndAdd(((Number) count).longValue());
                    }
                    Object time = m.get("timeMs");
                    if (time instanceof Number) {
                        _gcMs.getAndAdd(((Number) time).longValue());
                    }
                } else if (dp.name.startsWith("memory/") && dp.value instanceof Map) {
                    Map<Object, Object> m = (Map<Object, Object>) dp.value;
                    Object val = m.get("usedBytes");
                    if (val instanceof Number) {
                        MemMeasure mm = _memoryBytes.get(worker);
                        if (mm == null) {
                            mm = new MemMeasure();
                            MemMeasure tmp = _memoryBytes.putIfAbsent(worker, mm);
                            mm = tmp == null ? mm : tmp;
                        }
                        mm.update(((Number) val).longValue());
                    }
                }
            }
        }
    };
    metricServer.serve();
    String url = metricServer.getUrl();
    C cluster = new C(conf);
    conf.setNumWorkers(parallelism);
    conf.registerMetricsConsumer(org.apache.storm.metric.LoggingMetricsConsumer.class);
    conf.registerMetricsConsumer(org.apache.storm.metric.HttpForwardingMetricsConsumer.class, url, 1);
    Map<String, String> workerMetrics = new HashMap<String, String>();
    if (!cluster.isLocal()) {
        //sigar uses JNI and does not work in local mode
        workerMetrics.put("CPU", "org.apache.storm.metrics.sigar.CPUMetric");
    }
    conf.put(Config.TOPOLOGY_WORKER_METRICS, workerMetrics);
    conf.put(Config.TOPOLOGY_BUILTIN_METRICS_BUCKET_SIZE_SECS, 10);
    conf.put(Config.TOPOLOGY_WORKER_GC_CHILDOPTS, "-XX:+UseConcMarkSweepGC -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:NewSize=128m -XX:CMSInitiatingOccupancyFraction=70 -XX:-CMSConcurrentMTEnabled");
    conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, "-Xmx2g");
    TopologyBuilder builder = new TopologyBuilder();
    int numEach = 4 * parallelism;
    builder.setSpout("spout", new FastRandomSentenceSpout(ratePerSecond / numEach), numEach);
    builder.setBolt("split", new SplitSentence(), numEach).shuffleGrouping("spout");
    builder.setBolt("count", new WordCount(), numEach).fieldsGrouping("split", new Fields("word"));
    try {
        cluster.submitTopology(name, conf, builder.createTopology());
        for (int i = 0; i < numMins * 2; i++) {
            Thread.sleep(30 * 1000);
            printMetrics(cluster, name);
        }
    } finally {
        kill(cluster, name);
    }
    System.exit(0);
}