Examples with Fields backtype.storm.tuple.Fields used on opensource projects

Example 1 with Fields

use of backtype.storm.tuple.Fields in project storm by nathanmarz.

the class TridentTopology method multiReduce.

public Stream multiReduce(List<Fields> inputFields, List<GroupedStream> groupedStreams, GroupedMultiReducer function, Fields outputFields) {
    List<Fields> fullInputFields = new ArrayList<Fields>();
    List<Stream> streams = new ArrayList<Stream>();
    List<Fields> fullGroupFields = new ArrayList<Fields>();
    for (int i = 0; i < groupedStreams.size(); i++) {
        GroupedStream gs = groupedStreams.get(i);
        Fields groupFields = gs.getGroupFields();
        fullGroupFields.add(groupFields);
        streams.add(gs.toStream().partitionBy(groupFields));
        fullInputFields.add(TridentUtils.fieldsUnion(groupFields, inputFields.get(i)));
    }
    return multiReduce(fullInputFields, streams, new GroupedMultiReducerExecutor(function, fullGroupFields, inputFields), outputFields);
}

Example 2 with Fields

use of backtype.storm.tuple.Fields in project storm by nathanmarz.

the class TridentTopology method newStaticState.

public TridentState newStaticState(StateSpec spec) {
    String stateId = getUniqueStateId();
    Node n = new Node(getUniqueStreamId(), null, new Fields());
    n.stateInfo = new NodeStateInfo(stateId, spec);
    registerNode(n);
    return new TridentState(this, n);
}

Example 3 with Fields

use of backtype.storm.tuple.Fields in project storm by nathanmarz.

the class TridentTopology method completeDRPC.

private static void completeDRPC(DefaultDirectedGraph<Node, IndexedEdge> graph, Map<String, List<Node>> colocate, UniqueIdGen gen) {
    List<Set<Node>> connectedComponents = new ConnectivityInspector<Node, IndexedEdge>(graph).connectedSets();
    for (Set<Node> g : connectedComponents) {
        checkValidJoins(g);
    }
    TridentTopology helper = new TridentTopology(graph, colocate, gen);
    for (Set<Node> g : connectedComponents) {
        SpoutNode drpcNode = getDRPCSpoutNode(g);
        if (drpcNode != null) {
            Stream lastStream = new Stream(helper, null, getLastAddedNode(g));
            Stream s = new Stream(helper, null, drpcNode);
            helper.multiReduce(s.project(new Fields("return-info")).batchGlobal(), lastStream.batchGlobal(), new ReturnResultsReducer(), new Fields());
        }
    }
}

Example 4 with Fields

use of backtype.storm.tuple.Fields in project storm by nathanmarz.

the class TridentTopology method newDRPCStream.

private Stream newDRPCStream(DRPCSpout spout) {
    // TODO: consider adding a shuffle grouping after the spout to avoid so much routing of the args/return-info all over the place
    // (at least until its possible to just pack bolt logic into the spout itself)
    Node n = new SpoutNode(getUniqueStreamId(), TridentUtils.getSingleOutputStreamFields(spout), null, spout, SpoutNode.SpoutType.DRPC);
    Stream nextStream = addNode(n);
    // later on, this will be joined back with return-info and all the results
    return nextStream.project(new Fields("args"));
}

Example 5 with Fields

use of backtype.storm.tuple.Fields in project storm by nathanmarz.

the class ChainedAggregatorDeclarer method chainEnd.

public Stream chainEnd() {
    Fields[] inputFields = new Fields[_aggs.size()];
    Aggregator[] aggs = new Aggregator[_aggs.size()];
    int[] outSizes = new int[_aggs.size()];
    List<String> allOutFields = new ArrayList<String>();
    Set<String> allInFields = new HashSet<String>();
    for (int i = 0; i < _aggs.size(); i++) {
        AggSpec spec = _aggs.get(i);
        Fields infields = spec.inFields;
        if (infields == null)
            infields = new Fields();
        Fields outfields = spec.outFields;
        if (outfields == null)
            outfields = new Fields();
        inputFields[i] = infields;
        aggs[i] = spec.agg;
        outSizes[i] = outfields.size();
        allOutFields.addAll(outfields.toList());
        allInFields.addAll(infields.toList());
    }
    if (new HashSet(allOutFields).size() != allOutFields.size()) {
        throw new IllegalArgumentException("Output fields for chained aggregators must be distinct: " + allOutFields.toString());
    }
    Fields inFields = new Fields(new ArrayList<String>(allInFields));
    Fields outFields = new Fields(allOutFields);
    Aggregator combined = new ChainedAggregatorImpl(aggs, inputFields, new ComboList.Factory(outSizes));
    if (_type != AggType.FULL) {
        _stream = _stream.partitionAggregate(inFields, combined, outFields);
    }
    if (_type != AggType.PARTITION) {
        _stream = _globalScheme.aggPartition(_stream);
        BatchToPartition singleEmit = _globalScheme.singleEmitPartitioner();
        Aggregator toAgg = combined;
        if (singleEmit != null) {
            toAgg = new SingleEmitAggregator(combined, singleEmit);
        }
        // this assumes that inFields and outFields are the same for combineragg
        // assumption also made above
        _stream = _stream.partitionAggregate(inFields, toAgg, outFields);
    }
    return _stream.toStream();
}