Examples with InvalidTopologyException org.apache.storm.generated.InvalidTopologyException used on opensource projects

Example 6 with InvalidTopologyException

use of org.apache.storm.generated.InvalidTopologyException in project storm by apache.

the class TridentKafkaClientWordCountNamedTopics method run.

protected void run(String[] args) throws AlreadyAliveException, InvalidTopologyException, AuthorizationException, InterruptedException {
    if (args.length > 0 && Arrays.stream(args).anyMatch(option -> option.equals("-h"))) {
        System.out.printf("Usage: java %s [%s] [%s] [%s] [%s]\n", getClass().getName(), "broker_host:broker_port", "topic1", "topic2", "topology_name");
    } else {
        final String brokerUrl = args.length > 0 ? args[0] : KAFKA_LOCAL_BROKER;
        final String topic1 = args.length > 1 ? args[1] : TOPIC_1;
        final String topic2 = args.length > 2 ? args[2] : TOPIC_2;
        System.out.printf("Running with broker_url: [%s], topics: [%s, %s]\n", brokerUrl, topic1, topic2);
        Config tpConf = LocalSubmitter.defaultConfig(true);
        if (args.length == 4) {
            //Submit Remote
            // Producers
            StormSubmitter.submitTopology(topic1 + "-producer", tpConf, KafkaProducerTopology.newTopology(brokerUrl, topic1));
            StormSubmitter.submitTopology(topic2 + "-producer", tpConf, KafkaProducerTopology.newTopology(brokerUrl, topic2));
            // Consumer
            StormSubmitter.submitTopology("topics-consumer", tpConf, TridentKafkaConsumerTopology.newTopology(newKafkaTridentSpoutOpaque()));
            // Print results to console, which also causes the print filter in the consumer topology to print the results in the worker log
            Thread.sleep(2000);
            DrpcResultsPrinter.remoteClient().printResults(60, 1, TimeUnit.SECONDS);
        } else {
            //Submit Local
            final LocalSubmitter localSubmitter = LocalSubmitter.newInstance();
            final String topic1Tp = "topic1-producer";
            final String topic2Tp = "topic2-producer";
            final String consTpName = "topics-consumer";
            try {
                // Producers
                localSubmitter.submit(topic1Tp, tpConf, KafkaProducerTopology.newTopology(brokerUrl, topic1));
                localSubmitter.submit(topic2Tp, tpConf, KafkaProducerTopology.newTopology(brokerUrl, topic2));
                // Consumer
                try {
                    localSubmitter.submit(consTpName, tpConf, TridentKafkaConsumerTopology.newTopology(localSubmitter.getDrpc(), newKafkaTridentSpoutOpaque()));
                    // print
                    localSubmitter.printResults(15, 1, TimeUnit.SECONDS);
                } catch (Exception e) {
                    e.printStackTrace();
                }
            } finally {
                // kill
                localSubmitter.kill(topic1Tp);
                localSubmitter.kill(topic2Tp);
                localSubmitter.kill(consTpName);
                // shutdown
                localSubmitter.shutdown();
            }
        }
    }
    // Kill all the non daemon threads
    System.exit(0);
}

Example 7 with InvalidTopologyException

use of org.apache.storm.generated.InvalidTopologyException in project storm by apache.

the class BlobStoreAPIWordCountTopology method buildAndLaunchWordCountTopology.

public void buildAndLaunchWordCountTopology(String[] args) {
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("spout", new RandomSentenceSpout(), 5);
    builder.setBolt("split", new SplitSentence(), 8).shuffleGrouping("spout");
    builder.setBolt("filter", new FilterWords(), 6).shuffleGrouping("split");
    Config conf = new Config();
    conf.setDebug(true);
    try {
        conf.setNumWorkers(3);
        StormSubmitter.submitTopologyWithProgressBar(args[0], conf, builder.createTopology());
    } catch (InvalidTopologyException | AuthorizationException | AlreadyAliveException exp) {
        throw new RuntimeException(exp);
    }
}

Example 8 with InvalidTopologyException

use of org.apache.storm.generated.InvalidTopologyException in project storm by apache.

the class StormCommon method validateStructure.

public static void validateStructure(StormTopology topology) throws InvalidTopologyException {
    Map<String, Object> componentMap = allComponents(topology);
    for (Map.Entry<String, Object> entry : componentMap.entrySet()) {
        String componentId = entry.getKey();
        ComponentCommon common = getComponentCommon(entry.getValue());
        Map<GlobalStreamId, Grouping> inputs = common.get_inputs();
        for (Map.Entry<GlobalStreamId, Grouping> input : inputs.entrySet()) {
            String sourceStreamId = input.getKey().get_streamId();
            String sourceComponentId = input.getKey().get_componentId();
            if (!componentMap.keySet().contains(sourceComponentId)) {
                throw new InvalidTopologyException("Component: [" + componentId + "] subscribes from non-existent component [" + sourceComponentId + "]");
            }
            ComponentCommon sourceComponent = getComponentCommon(componentMap.get(sourceComponentId));
            if (!sourceComponent.get_streams().containsKey(sourceStreamId)) {
                throw new InvalidTopologyException("Component: [" + componentId + "] subscribes from non-existent stream: " + "[" + sourceStreamId + "] of component [" + sourceComponentId + "]");
            }
            Grouping grouping = input.getValue();
            if (Thrift.groupingType(grouping) == Grouping._Fields.FIELDS) {
                List<String> fields = new ArrayList<>(grouping.get_fields());
                Map<String, StreamInfo> streams = sourceComponent.get_streams();
                Set<String> sourceOutputFields = getStreamOutputFields(streams);
                fields.removeAll(sourceOutputFields);
                if (fields.size() != 0) {
                    throw new InvalidTopologyException("Component: [" + componentId + "] subscribes from stream: [" + sourceStreamId + "] of component " + "[" + sourceComponentId + "] + with non-existent fields: " + fields);
                }
            }
        }
    }
}

Example 9 with InvalidTopologyException

use of org.apache.storm.generated.InvalidTopologyException in project storm by apache.

the class StormCommon method validateBasic.

@SuppressWarnings("unchecked")
public static void validateBasic(StormTopology topology) throws InvalidTopologyException {
    validateIds(topology);
    for (StormTopology._Fields field : Thrift.getSpoutFields()) {
        Map<String, Object> spoutComponents = (Map<String, Object>) topology.getFieldValue(field);
        if (spoutComponents != null) {
            for (Object obj : spoutComponents.values()) {
                ComponentCommon common = getComponentCommon(obj);
                if (!isEmptyInputs(common)) {
                    throw new InvalidTopologyException("May not declare inputs for a spout");
                }
            }
        }
    }
    Map<String, Object> componentMap = allComponents(topology);
    for (Object componentObj : componentMap.values()) {
        Map conf = componentConf(componentObj);
        ComponentCommon common = getComponentCommon(componentObj);
        int parallelismHintNum = Thrift.getParallelismHint(common);
        Integer taskNum = Utils.getInt(conf.get(Config.TOPOLOGY_TASKS), 0);
        if (taskNum > 0 && parallelismHintNum <= 0) {
            throw new InvalidTopologyException("Number of executors must be greater than 0 when number of tasks is greater than 0");
        }
    }
}

Example 10 with InvalidTopologyException

use of org.apache.storm.generated.InvalidTopologyException in project storm by apache.

the class Nimbus method validateTopologySize.

private static void validateTopologySize(Map<String, Object> topoConf, Map<String, Object> nimbusConf, StormTopology topology) throws InvalidTopologyException {
    int workerCount = Utils.getInt(topoConf.get(Config.TOPOLOGY_WORKERS), 1);
    Integer allowedWorkers = Utils.getInt(nimbusConf.get(Config.NIMBUS_SLOTS_PER_TOPOLOGY), null);
    int executorsCount = 0;
    for (Object comp : StormCommon.allComponents(topology).values()) {
        executorsCount += StormCommon.numStartExecutors(comp);
    }
    Integer allowedExecutors = Utils.getInt(nimbusConf.get(Config.NIMBUS_EXECUTORS_PER_TOPOLOGY), null);
    if (allowedExecutors != null && executorsCount > allowedExecutors) {
        throw new InvalidTopologyException("Failed to submit topology. Topology requests more than " + allowedExecutors + " executors.");
    }
    if (allowedWorkers != null && workerCount > allowedWorkers) {
        throw new InvalidTopologyException("Failed to submit topology. Topology requests more than " + allowedWorkers + " workers.");
    }
}