Examples with GlobalStreamId backtype.storm.generated.GlobalStreamId used on opensource projects

Example 6 with GlobalStreamId

use of backtype.storm.generated.GlobalStreamId in project jstorm by alibaba.

the class StatefulWindowedBoltExecutor method initState.

@Override
public void initState(T state) {
    if (stateInitialized) {
        LOG.warn("State is already initialized. Ignoring initState");
        return;
    }
    statefulWindowedBolt.initState((T) state);
    // query the streamState for each input task stream and compute recoveryStates
    for (GlobalStreamId streamId : topologyContext.getThisSources().keySet()) {
        for (int taskId : topologyContext.getComponentTasks(streamId.get_componentId())) {
            WindowState windowState = streamState.get(new TaskStream(taskId, streamId));
            if (windowState != null) {
                recoveryStates.put(new TaskStream(taskId, streamId), windowState);
            }
        }
    }
    LOG.debug("recoveryStates {}", recoveryStates);
    stateInitialized = true;
    start();
}

Example 7 with GlobalStreamId

use of backtype.storm.generated.GlobalStreamId in project jstorm by alibaba.

the class TopologyBuilder method initCommon.

protected void initCommon(String id, IComponent component, Number parallelism) throws IllegalArgumentException {
    ComponentCommon common = new ComponentCommon();
    common.set_inputs(new HashMap<GlobalStreamId, Grouping>());
    if (parallelism != null) {
        int dop = parallelism.intValue();
        if (dop < 1) {
            throw new IllegalArgumentException("Parallelism must be positive.");
        }
        common.set_parallelism_hint(dop);
    } else {
        common.set_parallelism_hint(1);
    }
    Map conf = component.getComponentConfiguration();
    if (conf != null)
        common.set_json_conf(JSONValue.toJSONString(conf));
    _commons.put(id, common);
}

Example 8 with GlobalStreamId

use of backtype.storm.generated.GlobalStreamId in project jstorm by alibaba.

the class TransactionBolt method prepare.

@Override
public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
    this.conf = stormConf;
    this.topologyContext = context;
    this.topologyId = topologyContext.getTopologyId();
    this.taskId = topologyContext.getThisTaskId();
    this.componentId = topologyContext.getThisComponentId();
    this.upstreamTasks = TransactionCommon.getUpstreamTasks(componentId, topologyContext);
    this.downstreamTasks = TransactionCommon.getDownstreamTasks(componentId, topologyContext);
    this.topologyMasterId = context.getTopologyMasterId();
    LOG.info("TransactionBolt: upstreamTasks=" + upstreamTasks + ", downstreamTasks=" + downstreamTasks);
    this.outputCollector = new TransactionOutputCollector(this, collector);
    this.boltExecutor.prepare(conf, context, new OutputCollector(outputCollector));
    this.boltStatus = State.INIT;
    if (sysTopology == null) {
        try {
            sysTopology = Common.system_topology(stormConf, context.getRawTopology());
        } catch (InvalidTopologyException e) {
            LOG.error("Failed to build system topology", e);
            throw new RuntimeException(e);
        }
    }
    this.lastSuccessfulBatch = new ConcurrentHashMap<Integer, Long>();
    this.processingBatches = new HashMap<Integer, Map<Long, BatchTracker>>();
    Set<String> upstreamSpoutNames = TransactionCommon.getUpstreamSpouts(componentId, topologyContext);
    for (String spoutName : upstreamSpoutNames) {
        int groupId = TransactionCommon.groupIndex(topologyContext.getRawTopology(), spoutName);
        lastSuccessfulBatch.put(groupId, TransactionCommon.INIT_BATCH_ID);
        processingBatches.put(groupId, new HashMap<Long, BatchTracker>());
    }
    this.batchCache = new BatchCache(context, upstreamSpoutNames, sysTopology);
    this.kryoInput = new Input(1);
    this.streamIds = new SerializationFactory.IdDictionary(sysTopology);
    this.inputStreamIds = new HashSet<Integer>();
    Set<GlobalStreamId> inputs = topologyContext.getThisSources().keySet();
    for (GlobalStreamId stream : inputs) {
        inputStreamIds.add(streamIds.getStreamId(stream.get_componentId(), stream.get_streamId()));
    }
    for (String upstreamComponentId : TransactionCommon.getUpstreamComponents(componentId, topologyContext)) {
        inputStreamIds.add(streamIds.getStreamId(upstreamComponentId, TransactionCommon.BARRIER_STREAM_ID));
    }
    //LOG.info("Stream info prepare: streamIds={}, inputStreams={}, inputStreamIds={}", streamIds, inputs, inputStreamIds);
    startInitState();
}

Example 9 with GlobalStreamId

use of backtype.storm.generated.GlobalStreamId in project storm by nathanmarz.

the class TridentTopologyBuilder method buildTopology.

public StormTopology buildTopology() {
    TopologyBuilder builder = new TopologyBuilder();
    Map<GlobalStreamId, String> batchIdsForSpouts = fleshOutStreamBatchIds(false);
    Map<GlobalStreamId, String> batchIdsForBolts = fleshOutStreamBatchIds(true);
    Map<String, List<String>> batchesToCommitIds = new HashMap<String, List<String>>();
    Map<String, List<ITridentSpout>> batchesToSpouts = new HashMap<String, List<ITridentSpout>>();
    for (String id : _spouts.keySet()) {
        TransactionalSpoutComponent c = _spouts.get(id);
        if (c.spout instanceof IRichSpout) {
            //TODO: wrap this to set the stream name
            builder.setSpout(id, (IRichSpout) c.spout, c.parallelism);
        } else {
            String batchGroup = c.batchGroupId;
            if (!batchesToCommitIds.containsKey(batchGroup)) {
                batchesToCommitIds.put(batchGroup, new ArrayList<String>());
            }
            batchesToCommitIds.get(batchGroup).add(c.commitStateId);
            if (!batchesToSpouts.containsKey(batchGroup)) {
                batchesToSpouts.put(batchGroup, new ArrayList<ITridentSpout>());
            }
            batchesToSpouts.get(batchGroup).add((ITridentSpout) c.spout);
            BoltDeclarer scd = builder.setBolt(spoutCoordinator(id), new TridentSpoutCoordinator(c.commitStateId, (ITridentSpout) c.spout)).globalGrouping(masterCoordinator(c.batchGroupId), MasterBatchCoordinator.BATCH_STREAM_ID).globalGrouping(masterCoordinator(c.batchGroupId), MasterBatchCoordinator.SUCCESS_STREAM_ID);
            for (Map m : c.componentConfs) {
                scd.addConfigurations(m);
            }
            Map<String, TridentBoltExecutor.CoordSpec> specs = new HashMap();
            specs.put(c.batchGroupId, new CoordSpec());
            BoltDeclarer bd = builder.setBolt(id, new TridentBoltExecutor(new TridentSpoutExecutor(c.commitStateId, c.streamName, ((ITridentSpout) c.spout)), batchIdsForSpouts, specs), c.parallelism);
            bd.allGrouping(spoutCoordinator(id), MasterBatchCoordinator.BATCH_STREAM_ID);
            bd.allGrouping(masterCoordinator(batchGroup), MasterBatchCoordinator.SUCCESS_STREAM_ID);
            if (c.spout instanceof ICommitterTridentSpout) {
                bd.allGrouping(masterCoordinator(batchGroup), MasterBatchCoordinator.COMMIT_STREAM_ID);
            }
            for (Map m : c.componentConfs) {
                bd.addConfigurations(m);
            }
        }
    }
    for (String id : _batchPerTupleSpouts.keySet()) {
        SpoutComponent c = _batchPerTupleSpouts.get(id);
        SpoutDeclarer d = builder.setSpout(id, new RichSpoutBatchTriggerer((IRichSpout) c.spout, c.streamName, c.batchGroupId), c.parallelism);
        for (Map conf : c.componentConfs) {
            d.addConfigurations(conf);
        }
    }
    for (String batch : batchesToCommitIds.keySet()) {
        List<String> commitIds = batchesToCommitIds.get(batch);
        builder.setSpout(masterCoordinator(batch), new MasterBatchCoordinator(commitIds, batchesToSpouts.get(batch)));
    }
    for (String id : _bolts.keySet()) {
        Component c = _bolts.get(id);
        Map<String, CoordSpec> specs = new HashMap();
        for (GlobalStreamId s : getBoltSubscriptionStreams(id)) {
            String batch = batchIdsForBolts.get(s);
            if (!specs.containsKey(batch))
                specs.put(batch, new CoordSpec());
            CoordSpec spec = specs.get(batch);
            CoordType ct;
            if (_batchPerTupleSpouts.containsKey(s.get_componentId())) {
                ct = CoordType.single();
            } else {
                ct = CoordType.all();
            }
            spec.coords.put(s.get_componentId(), ct);
        }
        for (String b : c.committerBatches) {
            specs.get(b).commitStream = new GlobalStreamId(masterCoordinator(b), MasterBatchCoordinator.COMMIT_STREAM_ID);
        }
        BoltDeclarer d = builder.setBolt(id, new TridentBoltExecutor(c.bolt, batchIdsForBolts, specs), c.parallelism);
        for (Map conf : c.componentConfs) {
            d.addConfigurations(conf);
        }
        for (InputDeclaration inputDecl : c.declarations) {
            inputDecl.declare(d);
        }
        Map<String, Set<String>> batchToComponents = getBoltBatchToComponentSubscriptions(id);
        for (String b : batchToComponents.keySet()) {
            for (String comp : batchToComponents.get(b)) {
                d.directGrouping(comp, TridentBoltExecutor.COORD_STREAM(b));
            }
        }
        for (String b : c.committerBatches) {
            d.allGrouping(masterCoordinator(b), MasterBatchCoordinator.COMMIT_STREAM_ID);
        }
    }
    return builder.createTopology();
}

Example 10 with GlobalStreamId

use of backtype.storm.generated.GlobalStreamId in project storm by nathanmarz.

the class TridentTopologyBuilder method getBoltSubscriptionStreams.

List<GlobalStreamId> getBoltSubscriptionStreams(String id) {
    List<GlobalStreamId> ret = new ArrayList();
    Component c = _bolts.get(id);
    for (InputDeclaration d : c.declarations) {
        ret.add(new GlobalStreamId(d.getComponent(), d.getStream()));
    }
    return ret;
}