Search in sources :

Example 1 with SourceArgs

use of org.apache.storm.coordination.CoordinatedBolt.SourceArgs in project storm by apache.

the class TransactionalTopologyBuilder method buildTopologyBuilder.

public TopologyBuilder buildTopologyBuilder() {
    String coordinator = _spoutId + "/coordinator";
    TopologyBuilder builder = new TopologyBuilder();
    SpoutDeclarer declarer = builder.setSpout(coordinator, new TransactionalSpoutCoordinator(_spout));
    for (Map<String, Object> conf : _spoutConfs) {
        declarer.addConfigurations(conf);
    }
    declarer.addConfiguration(Config.TOPOLOGY_TRANSACTIONAL_ID, _id);
    BoltDeclarer emitterDeclarer = builder.setBolt(_spoutId, new CoordinatedBolt(new TransactionalSpoutBatchExecutor(_spout), null, null), _spoutParallelism).allGrouping(coordinator, TransactionalSpoutCoordinator.TRANSACTION_BATCH_STREAM_ID).addConfiguration(Config.TOPOLOGY_TRANSACTIONAL_ID, _id);
    if (_spout instanceof ICommitterTransactionalSpout) {
        emitterDeclarer.allGrouping(coordinator, TransactionalSpoutCoordinator.TRANSACTION_COMMIT_STREAM_ID);
    }
    for (String id : _bolts.keySet()) {
        Component component = _bolts.get(id);
        Map<String, SourceArgs> coordinatedArgs = new HashMap<String, SourceArgs>();
        for (String c : componentBoltSubscriptions(component)) {
            coordinatedArgs.put(c, SourceArgs.all());
        }
        IdStreamSpec idSpec = null;
        if (component.committer) {
            idSpec = IdStreamSpec.makeDetectSpec(coordinator, TransactionalSpoutCoordinator.TRANSACTION_COMMIT_STREAM_ID);
        }
        BoltDeclarer input = builder.setBolt(id, new CoordinatedBolt(component.bolt, coordinatedArgs, idSpec), component.parallelism);
        for (Map conf : component.componentConfs) {
            input.addConfigurations(conf);
        }
        for (String c : componentBoltSubscriptions(component)) {
            input.directGrouping(c, Constants.COORDINATED_STREAM_ID);
        }
        for (InputDeclaration d : component.declarations) {
            d.declare(input);
        }
        if (component.committer) {
            input.allGrouping(coordinator, TransactionalSpoutCoordinator.TRANSACTION_COMMIT_STREAM_ID);
        }
    }
    return builder;
}
Also used : TopologyBuilder(org.apache.storm.topology.TopologyBuilder) HashMap(java.util.HashMap) IdStreamSpec(org.apache.storm.coordination.CoordinatedBolt.IdStreamSpec) SourceArgs(org.apache.storm.coordination.CoordinatedBolt.SourceArgs) BoltDeclarer(org.apache.storm.topology.BoltDeclarer) SpoutDeclarer(org.apache.storm.topology.SpoutDeclarer) HashMap(java.util.HashMap) Map(java.util.Map) CoordinatedBolt(org.apache.storm.coordination.CoordinatedBolt)

Example 2 with SourceArgs

use of org.apache.storm.coordination.CoordinatedBolt.SourceArgs in project storm by apache.

the class BatchSubtopologyBuilder method extendTopology.

public void extendTopology(TopologyBuilder builder) {
    BoltDeclarer declarer = builder.setBolt(masterId, new CoordinatedBolt(masterBolt.bolt), masterBolt.parallelism);
    for (InputDeclaration decl : masterBolt.declarations) {
        decl.declare(declarer);
    }
    declarer.addConfigurations(masterBolt.componentConf);
    for (String id : bolts.keySet()) {
        Component component = bolts.get(id);
        Map<String, SourceArgs> coordinatedArgs = new HashMap<String, SourceArgs>();
        for (String c : componentBoltSubscriptions(component)) {
            SourceArgs source;
            if (c.equals(masterId)) {
                source = SourceArgs.single();
            } else {
                source = SourceArgs.all();
            }
            coordinatedArgs.put(c, source);
        }
        BoltDeclarer input = builder.setBolt(id, new CoordinatedBolt(component.bolt, coordinatedArgs, null), component.parallelism);
        for (SharedMemory request : component.sharedMemory) {
            input.addSharedMemory(request);
        }
        if (!component.componentConf.isEmpty()) {
            input.addConfigurations(component.componentConf);
        }
        for (String c : componentBoltSubscriptions(component)) {
            input.directGrouping(c, Constants.COORDINATED_STREAM_ID);
        }
        for (InputDeclaration d : component.declarations) {
            d.declare(input);
        }
    }
}
Also used : SourceArgs(org.apache.storm.coordination.CoordinatedBolt.SourceArgs) BoltDeclarer(org.apache.storm.topology.BoltDeclarer) HashMap(java.util.HashMap) SharedMemory(org.apache.storm.generated.SharedMemory)

Example 3 with SourceArgs

use of org.apache.storm.coordination.CoordinatedBolt.SourceArgs in project storm by apache.

the class LinearDRPCTopologyBuilder method createTopology.

private StormTopology createTopology(DRPCSpout spout) {
    final String SPOUT_ID = "spout";
    final String PREPARE_ID = "prepare-request";
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout(SPOUT_ID, spout);
    builder.setBolt(PREPARE_ID, new PrepareRequest()).noneGrouping(SPOUT_ID);
    int i = 0;
    for (; i < components.size(); i++) {
        Component component = components.get(i);
        Map<String, SourceArgs> source = new HashMap<String, SourceArgs>();
        if (i == 1) {
            source.put(boltId(i - 1), SourceArgs.single());
        } else if (i >= 2) {
            source.put(boltId(i - 1), SourceArgs.all());
        }
        IdStreamSpec idSpec = null;
        if (i == components.size() - 1 && component.bolt instanceof FinishedCallback) {
            idSpec = IdStreamSpec.makeDetectSpec(PREPARE_ID, PrepareRequest.ID_STREAM);
        }
        BoltDeclarer declarer = builder.setBolt(boltId(i), new CoordinatedBolt(component.bolt, source, idSpec), component.parallelism);
        for (SharedMemory request : component.sharedMemory) {
            declarer.addSharedMemory(request);
        }
        if (!component.componentConf.isEmpty()) {
            declarer.addConfigurations(component.componentConf);
        }
        if (idSpec != null) {
            declarer.fieldsGrouping(idSpec.getGlobalStreamId().get_componentId(), PrepareRequest.ID_STREAM, new Fields("request"));
        }
        if (i == 0 && component.declarations.isEmpty()) {
            declarer.noneGrouping(PREPARE_ID, PrepareRequest.ARGS_STREAM);
        } else {
            String prevId;
            if (i == 0) {
                prevId = PREPARE_ID;
            } else {
                prevId = boltId(i - 1);
            }
            for (InputDeclaration declaration : component.declarations) {
                declaration.declare(prevId, declarer);
            }
        }
        if (i > 0) {
            declarer.directGrouping(boltId(i - 1), Constants.COORDINATED_STREAM_ID);
        }
    }
    IRichBolt lastBolt = components.get(components.size() - 1).bolt;
    OutputFieldsGetter getter = new OutputFieldsGetter();
    lastBolt.declareOutputFields(getter);
    Map<String, StreamInfo> streams = getter.getFieldsDeclaration();
    if (streams.size() != 1) {
        throw new RuntimeException("Must declare exactly one stream from last bolt in LinearDRPCTopology");
    }
    String outputStream = streams.keySet().iterator().next();
    List<String> fields = streams.get(outputStream).get_output_fields();
    if (fields.size() != 2) {
        throw new RuntimeException("Output stream of last component in LinearDRPCTopology must contain exactly two fields. " + "The first should be the request id, and the second should be the result.");
    }
    builder.setBolt(boltId(i), new JoinResult(PREPARE_ID)).fieldsGrouping(boltId(i - 1), outputStream, new Fields(fields.get(0))).fieldsGrouping(PREPARE_ID, PrepareRequest.RETURN_STREAM, new Fields("request"));
    i++;
    builder.setBolt(boltId(i), new ReturnResults()).noneGrouping(boltId(i - 1));
    return builder.createTopology();
}
Also used : IRichBolt(org.apache.storm.topology.IRichBolt) TopologyBuilder(org.apache.storm.topology.TopologyBuilder) HashMap(java.util.HashMap) OutputFieldsGetter(org.apache.storm.topology.OutputFieldsGetter) IdStreamSpec(org.apache.storm.coordination.CoordinatedBolt.IdStreamSpec) SourceArgs(org.apache.storm.coordination.CoordinatedBolt.SourceArgs) Fields(org.apache.storm.tuple.Fields) BoltDeclarer(org.apache.storm.topology.BoltDeclarer) StreamInfo(org.apache.storm.generated.StreamInfo) SharedMemory(org.apache.storm.generated.SharedMemory) FinishedCallback(org.apache.storm.coordination.CoordinatedBolt.FinishedCallback) CoordinatedBolt(org.apache.storm.coordination.CoordinatedBolt)

Aggregations

HashMap (java.util.HashMap)3 SourceArgs (org.apache.storm.coordination.CoordinatedBolt.SourceArgs)3 BoltDeclarer (org.apache.storm.topology.BoltDeclarer)3 CoordinatedBolt (org.apache.storm.coordination.CoordinatedBolt)2 IdStreamSpec (org.apache.storm.coordination.CoordinatedBolt.IdStreamSpec)2 SharedMemory (org.apache.storm.generated.SharedMemory)2 TopologyBuilder (org.apache.storm.topology.TopologyBuilder)2 Map (java.util.Map)1 FinishedCallback (org.apache.storm.coordination.CoordinatedBolt.FinishedCallback)1 StreamInfo (org.apache.storm.generated.StreamInfo)1 IRichBolt (org.apache.storm.topology.IRichBolt)1 OutputFieldsGetter (org.apache.storm.topology.OutputFieldsGetter)1 SpoutDeclarer (org.apache.storm.topology.SpoutDeclarer)1 Fields (org.apache.storm.tuple.Fields)1