Examples with Fields backtype.storm.tuple.Fields used on opensource projects

Example 96 with Fields

use of backtype.storm.tuple.Fields in project jstorm by alibaba.

the class TridentMinMaxOfDevicesTopology method test.

public static void test() {
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("spout", new InOrderSpout(), 8);
    builder.setBolt("count", new Check(), 8).fieldsGrouping("spout", new Fields("c1"));
    conf.setMaxSpoutPending(20);
    String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\.");
    String topologyName = className[className.length - 1];
    if (isLocal) {
        drpc = new LocalDRPC();
    }
    try {
        JStormHelper.runTopology(buildDevicesTopology(), topologyName, conf, 60, new JStormHelper.CheckAckedFail(conf), isLocal);
    } catch (Exception e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
        Assert.fail("Failed");
    }
}

Example 97 with Fields

use of backtype.storm.tuple.Fields in project jstorm by alibaba.

the class TridentMinMaxOfDevicesTopology method buildVehiclesTopology.

/**
 * Creates a topology which demonstrates min/max operations on tuples of
 * stream which contain vehicle and driver fields with values
 * {@link TridentMinMaxOfDevicesTopology.Vehicle} and
 * {@link TridentMinMaxOfDevicesTopology.Driver} respectively.
 */
public static StormTopology buildVehiclesTopology() {
    Fields driverField = new Fields(Driver.FIELD_NAME);
    Fields vehicleField = new Fields(Vehicle.FIELD_NAME);
    Fields allFields = new Fields(Vehicle.FIELD_NAME, Driver.FIELD_NAME);
    FixedBatchSpout spout = new FixedBatchSpout(allFields, 10, Vehicle.generateVehicles(20));
    spout.setCycle(true);
    TridentTopology topology = new TridentTopology();
    Stream vehiclesStream = topology.newStream("spout1", spout).each(allFields, new Debug("##### vehicles"));
    Stream slowVehiclesStream = vehiclesStream.min(new SpeedComparator()).each(vehicleField, new Debug("#### slowest vehicle"));
    Stream slowDriversStream = slowVehiclesStream.project(driverField).each(driverField, new Debug("##### slowest driver"));
    vehiclesStream.max(new SpeedComparator()).each(vehicleField, new Debug("#### fastest vehicle")).project(driverField).each(driverField, new Debug("##### fastest driver"));
    vehiclesStream.max(new EfficiencyComparator()).each(vehicleField, new Debug("#### efficient vehicle"));
    return topology.build();
}

Example 98 with Fields

use of backtype.storm.tuple.Fields in project jstorm by alibaba.

the class KvStatefulBoltExecutor method prepare.

@Override
public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
    createState(context);
    prepare(stormConf, context, collector, keyRangeState);
    Map<GlobalStreamId, Grouping> sources = context.getSources(context.getThisComponentId());
    for (Map.Entry<GlobalStreamId, Grouping> entry : sources.entrySet()) {
        GlobalStreamId stream = entry.getKey();
        Grouping grouping = entry.getValue();
        Grouping._Fields groupingFields = Thrift.groupingType(grouping);
        if (Grouping._Fields.FIELDS.equals(groupingFields)) {
            Fields fields = new Fields(Thrift.fieldGrouping(grouping));
            fieldGrouping.put(stream.get_streamId(), fields);
        }
    }
    LOG.info("Source fieldgrouping streams: {}", fieldGrouping);
}

Example 99 with Fields

use of backtype.storm.tuple.Fields in project jstorm by alibaba.

the class MkGrouper method parseGroupType.

private GrouperType parseGroupType(WorkerData workerData) {
    GrouperType grouperType = null;
    if (Grouping._Fields.FIELDS.equals(fields)) {
        if (Thrift.isGlobalGrouping(thriftGrouping)) {
            // global grouping, just send tuple to first task
            grouperType = GrouperType.global;
        } else {
            List<String> fields_group = Thrift.fieldGrouping(thriftGrouping);
            Fields fields = new Fields(fields_group);
            Map conf = topologyContext.getStormConf();
            boolean enableKeyRangeHash = ConfigExtension.isEnableKeyRangeFieldGroup(conf);
            if (enableKeyRangeHash)
                fieldsGrouper = new MkKeyRangeFieldsGrouper(conf, outFields, fields, outTasks);
            else
                fieldsGrouper = new MkFieldsGrouper(outFields, fields, outTasks);
            // hashcode by fields
            grouperType = GrouperType.fields;
        }
    } else if (Grouping._Fields.ALL.equals(fields)) {
        // send to every task
        grouperType = GrouperType.all;
    } else if (Grouping._Fields.SHUFFLE.equals(fields)) {
        grouperType = GrouperType.shuffle;
        shuffer = new MkShuffer(topologyContext.getThisComponentId(), targetComponent, workerData);
    } else if (Grouping._Fields.NONE.equals(fields)) {
        // random send one task
        this.random = new Random();
        grouperType = GrouperType.none;
    } else if (Grouping._Fields.CUSTOM_OBJECT.equals(fields)) {
        // user custom grouping by JavaObject
        JavaObject jobj = thriftGrouping.get_custom_object();
        CustomStreamGrouping g = Thrift.instantiateJavaObject(jobj);
        int myTaskId = topologyContext.getThisTaskId();
        String componentId = topologyContext.getComponentId(myTaskId);
        GlobalStreamId stream = new GlobalStreamId(componentId, streamId);
        customGrouper = new MkCustomGrouper(topologyContext, g, stream, outTasks, myTaskId);
        grouperType = GrouperType.custom_obj;
    } else if (Grouping._Fields.CUSTOM_SERIALIZED.equals(fields)) {
        // user custom group by serialized Object
        byte[] obj = thriftGrouping.get_custom_serialized();
        CustomStreamGrouping g = (CustomStreamGrouping) Utils.javaDeserialize(obj);
        int myTaskId = topologyContext.getThisTaskId();
        String componentId = topologyContext.getComponentId(myTaskId);
        GlobalStreamId stream = new GlobalStreamId(componentId, streamId);
        customGrouper = new MkCustomGrouper(topologyContext, g, stream, outTasks, myTaskId);
        grouperType = GrouperType.custom_serialized;
    } else if (Grouping._Fields.DIRECT.equals(fields)) {
        // directly send to a special task
        grouperType = GrouperType.direct;
    } else if (Grouping._Fields.LOCAL_OR_SHUFFLE.equals(fields)) {
        grouperType = GrouperType.shuffle;
        shuffer = new MkShuffer(topologyContext.getThisComponentId(), targetComponent, workerData);
    } else if (Grouping._Fields.LOCAL_FIRST.equals(fields)) {
        grouperType = GrouperType.shuffle;
        shuffer = new MkShuffer(topologyContext.getThisComponentId(), targetComponent, workerData);
    }
    return grouperType;
}

Example 100 with Fields

use of backtype.storm.tuple.Fields in project jstorm by alibaba.

the class TransactionBolt method declareOutputFields.

@Override
public void declareOutputFields(OutputFieldsDeclarer declarer) {
    boltExecutor.declareOutputFields(declarer);
    Map<String, StreamInfo> streams = ((OutputFieldsGetter) declarer).getFieldsDeclaration();
    if (streams.size() > 0) {
        declarer.declareStream(TransactionCommon.BARRIER_STREAM_ID, new Fields(TransactionCommon.BARRIER_SNAPSHOT_FIELD));
    } else {
        isEndBolt = true;
    }
}