Examples with SpoutSpec org.apache.storm.generated.SpoutSpec used on opensource projects

Example 26 with SpoutSpec

use of org.apache.storm.generated.SpoutSpec in project storm by apache.

the class TestTopologyAnonymizerUtils method readAndAnonymizeTopology.

private StormTopology readAndAnonymizeTopology(String resource, List<String> errors) {
    StormTopology stormTopology;
    try {
        stormTopology = Utils.deserialize(getResourceAsBytes(resource), StormTopology.class);
    } catch (Exception ex) {
        String err = String.format("Cannot read topology from resource %s", resource);
        errors.add(err);
        LOG.error(err, ex);
        return null;
    }
    Map<String, String> renameMap = new HashMap<>();
    if (stormTopology.get_spouts() != null) {
        for (String name : stormTopology.get_spouts().keySet()) {
            String newName = String.format("Spout-%d", renameMap.size());
            renameMap.putIfAbsent(name, newName);
        }
    }
    int spoutCnt = renameMap.size();
    if (stormTopology.get_bolts() != null) {
        for (String name : stormTopology.get_bolts().keySet()) {
            String newName = String.format("Bolt-%d", renameMap.size() - spoutCnt);
            renameMap.putIfAbsent(name, newName);
        }
    }
    int boltCnt = renameMap.size() - spoutCnt;
    // rename components
    StormTopology retVal = stormTopology.deepCopy();
    if (spoutCnt > 0) {
        Map<String, SpoutSpec> spouts = retVal.get_spouts();
        for (String name : renameMap.keySet()) {
            if (spouts.containsKey(name)) {
                spouts.put(renameMap.get(name), spouts.remove(name));
            }
        }
        retVal.get_spouts().values().forEach(spec -> {
            for (GlobalStreamId globalId : spec.get_common().get_inputs().keySet()) {
                if (renameMap.containsKey(globalId.get_componentId())) {
                    globalId.set_componentId(renameMap.get(globalId.get_componentId()));
                }
            }
        });
    }
    if (boltCnt > 0) {
        Map<String, Bolt> bolts = retVal.get_bolts();
        for (String name : renameMap.keySet()) {
            if (bolts.containsKey(name)) {
                bolts.put(renameMap.get(name), bolts.remove(name));
            }
        }
        retVal.get_bolts().values().forEach(spec -> {
            for (GlobalStreamId globalId : spec.get_common().get_inputs().keySet()) {
                if (renameMap.containsKey(globalId.get_componentId())) {
                    globalId.set_componentId(renameMap.get(globalId.get_componentId()));
                }
            }
        });
    }
    return retVal;
}

Example 27 with SpoutSpec

use of org.apache.storm.generated.SpoutSpec in project storm by apache.

the class TestUtilsForResourceAwareScheduler method genExecsAndComps.

public static Map<ExecutorDetails, String> genExecsAndComps(StormTopology topology) {
    Map<ExecutorDetails, String> retMap = new HashMap<>();
    int startTask = 0;
    int endTask = 0;
    for (Map.Entry<String, SpoutSpec> entry : topology.get_spouts().entrySet()) {
        SpoutSpec spout = entry.getValue();
        String spoutId = entry.getKey();
        int spoutParallelism = spout.get_common().get_parallelism_hint();
        for (int i = 0; i < spoutParallelism; i++) {
            retMap.put(new ExecutorDetails(startTask, endTask), spoutId);
            startTask++;
            endTask++;
        }
    }
    for (Map.Entry<String, Bolt> entry : topology.get_bolts().entrySet()) {
        String boltId = entry.getKey();
        Bolt bolt = entry.getValue();
        int boltParallelism = bolt.get_common().get_parallelism_hint();
        for (int i = 0; i < boltParallelism; i++) {
            retMap.put(new ExecutorDetails(startTask, endTask), boltId);
            startTask++;
            endTask++;
        }
    }
    return retMap;
}

Example 28 with SpoutSpec

use of org.apache.storm.generated.SpoutSpec in project storm by apache.

the class StormCommon method addAcker.

@SuppressWarnings("unchecked")
public static void addAcker(Map<String, Object> conf, StormTopology topology) {
    Map<String, StreamInfo> outputStreams = new HashMap<String, StreamInfo>();
    outputStreams.put(Acker.ACKER_ACK_STREAM_ID, Thrift.directOutputFields(Arrays.asList("id", "time-delta-ms")));
    outputStreams.put(Acker.ACKER_FAIL_STREAM_ID, Thrift.directOutputFields(Arrays.asList("id", "time-delta-ms")));
    outputStreams.put(Acker.ACKER_RESET_TIMEOUT_STREAM_ID, Thrift.directOutputFields(Arrays.asList("id", "time-delta-ms")));
    Map<String, Object> ackerConf = new HashMap<>();
    int ackerNum = ObjectReader.getInt(conf.get(Config.TOPOLOGY_ACKER_EXECUTORS), ObjectReader.getInt(conf.get(Config.TOPOLOGY_WORKERS)));
    ackerConf.put(Config.TOPOLOGY_TASKS, ackerNum);
    ackerConf.put(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, ObjectReader.getInt(conf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS)));
    Map<GlobalStreamId, Grouping> inputs = ackerInputs(topology);
    Bolt acker = Thrift.prepareSerializedBoltDetails(inputs, makeAckerBolt(), outputStreams, ackerNum, ackerConf);
    for (Bolt bolt : topology.get_bolts().values()) {
        ComponentCommon common = bolt.get_common();
        common.put_to_streams(Acker.ACKER_ACK_STREAM_ID, Thrift.outputFields(Arrays.asList("id", "ack-val")));
        common.put_to_streams(Acker.ACKER_FAIL_STREAM_ID, Thrift.outputFields(Arrays.asList("id")));
        common.put_to_streams(Acker.ACKER_RESET_TIMEOUT_STREAM_ID, Thrift.outputFields(Arrays.asList("id")));
    }
    for (SpoutSpec spout : topology.get_spouts().values()) {
        ComponentCommon common = spout.get_common();
        Map<String, Object> spoutConf = componentConf(spout);
        spoutConf.put(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, ObjectReader.getInt(conf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS)));
        common.set_json_conf(JSONValue.toJSONString(spoutConf));
        common.put_to_streams(Acker.ACKER_INIT_STREAM_ID, Thrift.outputFields(Arrays.asList("id", "init-val", "spout-task")));
        common.put_to_inputs(Utils.getGlobalStreamId(Acker.ACKER_COMPONENT_ID, Acker.ACKER_ACK_STREAM_ID), Thrift.prepareDirectGrouping());
        common.put_to_inputs(Utils.getGlobalStreamId(Acker.ACKER_COMPONENT_ID, Acker.ACKER_FAIL_STREAM_ID), Thrift.prepareDirectGrouping());
        common.put_to_inputs(Utils.getGlobalStreamId(Acker.ACKER_COMPONENT_ID, Acker.ACKER_RESET_TIMEOUT_STREAM_ID), Thrift.prepareDirectGrouping());
    }
    topology.put_to_bolts(Acker.ACKER_COMPONENT_ID, acker);
}

Example 29 with SpoutSpec

use of org.apache.storm.generated.SpoutSpec in project storm by apache.

the class Executor method getExecutorType.

private static String getExecutorType(WorkerTopologyContext workerTopologyContext, String componentId) {
    StormTopology topology = workerTopologyContext.getRawTopology();
    Map<String, SpoutSpec> spouts = topology.get_spouts();
    Map<String, Bolt> bolts = topology.get_bolts();
    if (spouts.containsKey(componentId)) {
        return ClientStatsUtil.SPOUT;
    } else if (bolts.containsKey(componentId)) {
        return ClientStatsUtil.BOLT;
    } else {
        throw new RuntimeException("Could not find " + componentId + " in " + topology);
    }
}

Example 30 with SpoutSpec

use of org.apache.storm.generated.SpoutSpec in project storm by apache.

the class StrictTopologyValidator method validate.

@Override
public void validate(String topologyName, Map topologyConf, StormTopology topology) throws InvalidTopologyException {
    if (topologyName.contains(".")) {
        throw new WrappedInvalidTopologyException(String.format("Topology name '%s' contains illegal character '.'", topologyName));
    }
    Map<String, SpoutSpec> spouts = topology.get_spouts();
    for (String spoutName : spouts.keySet()) {
        if (spoutName.contains(".")) {
            throw new WrappedInvalidTopologyException(String.format("Spout name '%s' contains illegal character '.'", spoutName));
        }
        SpoutSpec spoutSpec = spouts.get(spoutName);
        for (String streamName : spoutSpec.get_common().get_streams().keySet()) {
            if (streamName.contains(".")) {
                throw new WrappedInvalidTopologyException(String.format("Stream name '%s' contains illegal character '.'", streamName));
            }
        }
    }
    Map<String, Bolt> bolts = topology.get_bolts();
    for (String boltName : bolts.keySet()) {
        if (boltName.contains(".")) {
            throw new WrappedInvalidTopologyException(String.format("Bolt name '%s' contains illegal character '.'", boltName));
        }
        Bolt bolt = bolts.get(boltName);
        for (String streamName : bolt.get_common().get_streams().keySet()) {
            if (streamName.contains(".")) {
                throw new WrappedInvalidTopologyException(String.format("Stream name '%s' contains illegal character '.'", streamName));
            }
        }
    }
}