Examples with StormTopology backtype.storm.generated.StormTopology used on opensource projects

Example 6 with StormTopology

use of backtype.storm.generated.StormTopology in project jstorm by alibaba.

the class TopologyBuilder method createTopology.

public StormTopology createTopology() {
    Map<String, Bolt> boltSpecs = new HashMap<>();
    Map<String, SpoutSpec> spoutSpecs = new HashMap<>();
    maybeAddCheckpointSpout();
    for (String boltId : _bolts.keySet()) {
        IRichBolt bolt = _bolts.get(boltId);
        bolt = maybeAddCheckpointTupleForwarder(bolt);
        ComponentCommon common = getComponentCommon(boltId, bolt);
        try {
            maybeAddCheckpointInputs(common);
            boltSpecs.put(boltId, new Bolt(ComponentObject.serialized_java(Utils.javaSerialize(bolt)), common));
        } catch (RuntimeException wrapperCause) {
            if (wrapperCause.getCause() != null && NotSerializableException.class.equals(wrapperCause.getCause().getClass())) {
                throw new IllegalStateException("Bolt '" + boltId + "' contains a non-serializable field of type " + wrapperCause.getCause().getMessage() + ", " + "which was instantiated prior to topology creation. " + wrapperCause.getCause().getMessage() + " " + "should be instantiated within the prepare method of '" + boltId + " at the earliest.", wrapperCause);
            }
            throw wrapperCause;
        }
    }
    for (String spoutId : _spouts.keySet()) {
        IRichSpout spout = _spouts.get(spoutId);
        ComponentCommon common = getComponentCommon(spoutId, spout);
        try {
            spoutSpecs.put(spoutId, new SpoutSpec(ComponentObject.serialized_java(Utils.javaSerialize(spout)), common));
        } catch (RuntimeException wrapperCause) {
            if (wrapperCause.getCause() != null && NotSerializableException.class.equals(wrapperCause.getCause().getClass())) {
                throw new IllegalStateException("Spout '" + spoutId + "' contains a non-serializable field of type " + wrapperCause.getCause().getMessage() + ", " + "which was instantiated prior to topology creation. " + wrapperCause.getCause().getMessage() + " " + "should be instantiated within the prepare method of '" + spoutId + " at the earliest.", wrapperCause);
            }
            throw wrapperCause;
        }
    }
    StormTopology stormTopology = new StormTopology(spoutSpecs, boltSpecs, new HashMap<String, StateSpoutSpec>());
    //stormTopology.set_worker_hooks(_workerHooks);
    return stormTopology;
}

Example 7 with StormTopology

use of backtype.storm.generated.StormTopology in project jstorm by alibaba.

the class TCKTest method testTopologySourceWithConfigMethods.

@Test
public void testTopologySourceWithConfigMethods() throws Exception {
    TopologyDef topologyDef = FluxParser.parseResource("/configs/config-methods-test.yaml", false, true, null, false);
    assertTrue(topologyDef.validate());
    Config conf = FluxBuilder.buildConfig(topologyDef);
    ExecutionContext context = new ExecutionContext(topologyDef, conf);
    StormTopology topology = FluxBuilder.buildTopology(context);
    assertNotNull(topology);
    topology.validate();
    // make sure the property was actually set
    TestBolt bolt = (TestBolt) context.getBolt("bolt-1");
    assertTrue(bolt.getFoo().equals("foo"));
    assertTrue(bolt.getBar().equals("bar"));
    assertTrue(bolt.getFooBar().equals("foobar"));
}

Example 8 with StormTopology

use of backtype.storm.generated.StormTopology in project jstorm by alibaba.

the class Flux method runCli.

private static void runCli(CommandLine cmd) throws Exception {
    if (!cmd.hasOption(OPTION_NO_SPLASH)) {
        printSplash();
    }
    boolean dumpYaml = cmd.hasOption("dump-yaml");
    TopologyDef topologyDef = null;
    String filePath = (String) cmd.getArgList().get(0);
    // TODO conditionally load properties from a file our resource
    String filterProps = null;
    if (cmd.hasOption(OPTION_FILTER)) {
        filterProps = cmd.getOptionValue(OPTION_FILTER);
    }
    boolean envFilter = cmd.hasOption(OPTION_ENV_FILTER);
    if (cmd.hasOption(OPTION_RESOURCE)) {
        printf("Parsing classpath resource: %s", filePath);
        topologyDef = FluxParser.parseResource(filePath, dumpYaml, true, filterProps, envFilter);
    } else {
        printf("Parsing file: %s", new File(filePath).getAbsolutePath());
        topologyDef = FluxParser.parseFile(filePath, dumpYaml, true, filterProps, envFilter);
    }
    String topologyName = topologyDef.getName();
    // merge contents of `config` into topology config
    Config conf = FluxBuilder.buildConfig(topologyDef);
    ExecutionContext context = new ExecutionContext(topologyDef, conf);
    StormTopology topology = FluxBuilder.buildTopology(context);
    if (!cmd.hasOption(OPTION_NO_DETAIL)) {
        printTopologyInfo(context);
    }
    if (!cmd.hasOption(OPTION_DRY_RUN)) {
        if (cmd.hasOption(OPTION_REMOTE)) {
            LOG.info("Running remotely...");
            try {
                // should the topology be active or inactive
                SubmitOptions submitOptions = null;
                if (cmd.hasOption(OPTION_INACTIVE)) {
                    LOG.info("Deploying topology in an INACTIVE state...");
                    submitOptions = new SubmitOptions(TopologyInitialStatus.INACTIVE);
                } else {
                    LOG.info("Deploying topology in an ACTIVE state...");
                    submitOptions = new SubmitOptions(TopologyInitialStatus.ACTIVE);
                }
                StormSubmitter.submitTopology(topologyName, conf, topology, submitOptions);
            } catch (Exception e) {
                LOG.warn("Unable to deploy topology to remote cluster.", e);
            }
        } else {
            LOG.info("Running in local mode...");
            String sleepStr = cmd.getOptionValue(OPTION_SLEEP);
            Long sleepTime = DEFAULT_LOCAL_SLEEP_TIME;
            if (sleepStr != null) {
                sleepTime = Long.parseLong(sleepStr);
            }
            LOG.debug("Sleep time: {}", sleepTime);
            LocalCluster cluster = null;
            // in-process or external zookeeper
            if (cmd.hasOption(OPTION_ZOOKEEPER)) {
                String zkStr = cmd.getOptionValue(OPTION_ZOOKEEPER);
                LOG.info("Using ZooKeeper at '{}' instead of in-process one.", zkStr);
                long zkPort = DEFAULT_ZK_PORT;
                String zkHost = null;
                if (zkStr.contains(":")) {
                    String[] hostPort = zkStr.split(":");
                    zkHost = hostPort[0];
                    zkPort = hostPort.length > 1 ? Long.parseLong(hostPort[1]) : DEFAULT_ZK_PORT;
                } else {
                    zkHost = zkStr;
                }
                // the following constructor is only available in 0.9.3 and later
                /*                    try {
                        cluster = new LocalCluster(zkHost, zkPort);
                    } catch (NoSuchMethodError e){
                        LOG.error("The --zookeeper option can only be used with Apache Storm 0.9.3 and later.");
                        System.exit(1);
                    }*/
                LOG.error("sorry, jstorm don't support this operation!!!");
                System.exit(1);
            } else {
                cluster = new LocalCluster();
            }
            cluster.submitTopology(topologyName, conf, topology);
            Utils.sleep(sleepTime);
            cluster.killTopology(topologyName);
            cluster.shutdown();
        }
    }
}

Example 9 with StormTopology

use of backtype.storm.generated.StormTopology in project jstorm by alibaba.

the class FluxBuilder method buildExternalTopology.

private static StormTopology buildExternalTopology(ObjectDef def, ExecutionContext context) throws ClassNotFoundException, IllegalAccessException, InstantiationException, NoSuchMethodException, InvocationTargetException {
    Object topologySource = buildObject(def, context);
    String methodName = context.getTopologyDef().getTopologySource().getMethodName();
    Method getTopology = findGetTopologyMethod(topologySource, methodName);
    if (getTopology.getParameterTypes()[0].equals(Config.class)) {
        Config config = new Config();
        config.putAll(context.getTopologyDef().getConfig());
        return (StormTopology) getTopology.invoke(topologySource, config);
    } else {
        return (StormTopology) getTopology.invoke(topologySource, context.getTopologyDef().getConfig());
    }
}

Example 10 with StormTopology

use of backtype.storm.generated.StormTopology in project jstorm by alibaba.

the class FluxBuilder method buildTopology.

/**
     * Given a topology definition, return a Storm topology that can be run either locally or remotely.
     *
     * @param context
     * @return
     * @throws IllegalAccessException
     * @throws InstantiationException
     * @throws ClassNotFoundException
     * @throws NoSuchMethodException
     * @throws InvocationTargetException
     */
public static StormTopology buildTopology(ExecutionContext context) throws IllegalAccessException, InstantiationException, ClassNotFoundException, NoSuchMethodException, InvocationTargetException {
    StormTopology topology = null;
    TopologyDef topologyDef = context.getTopologyDef();
    if (!topologyDef.validate()) {
        throw new IllegalArgumentException("Invalid topology config. Spouts, bolts and streams cannot be " + "defined in the same configuration as a topologySource.");
    }
    // build components that may be referenced by spouts, bolts, etc.
    // the map will be a String --> Object where the object is a fully
    // constructed class instance
    buildComponents(context);
    if (topologyDef.isDslTopology()) {
        // This is a DSL (YAML, etc.) topology...
        LOG.info("Detected DSL topology...");
        TopologyBuilder builder = new TopologyBuilder();
        // create spouts
        buildSpouts(context, builder);
        // we need to be able to lookup bolts by id, then switch based
        // on whether they are IBasicBolt or IRichBolt instances
        buildBolts(context);
        // process stream definitions
        buildStreamDefinitions(context, builder);
        topology = builder.createTopology();
    } else {
        // user class supplied...
        // this also provides a bridge to Trident...
        LOG.info("A topology source has been specified...");
        ObjectDef def = topologyDef.getTopologySource();
        topology = buildExternalTopology(def, context);
    }
    return topology;
}