Examples with TopologyDef org.apache.storm.flux.model.TopologyDef used on opensource projects

Example 11 with TopologyDef

use of org.apache.storm.flux.model.TopologyDef in project storm by apache.

the class TCKTest method testTopologySourceWithConfigParam.

@Test
public void testTopologySourceWithConfigParam() throws Exception {
    TopologyDef topologyDef = FluxParser.parseResource("/configs/existing-topology-reflection-config.yaml", false, true, null, false);
    assertTrue(topologyDef.validate());
    Config conf = FluxBuilder.buildConfig(topologyDef);
    ExecutionContext context = new ExecutionContext(topologyDef, conf);
    StormTopology topology = FluxBuilder.buildTopology(context);
    assertNotNull(topology);
    topology.validate();
}

Example 12 with TopologyDef

use of org.apache.storm.flux.model.TopologyDef in project storm by apache.

the class FluxParser method parseFile.

// TODO refactor input stream processing (see parseResource() method).
public static TopologyDef parseFile(String inputFile, boolean dumpYaml, boolean processIncludes, String propertiesFile, boolean envSub) throws IOException {
    FileInputStream in = new FileInputStream(inputFile);
    TopologyDef topology = parseInputStream(in, dumpYaml, processIncludes, propertiesFile, envSub);
    in.close();
    return topology;
}

Example 13 with TopologyDef

use of org.apache.storm.flux.model.TopologyDef in project storm by apache.

the class FluxParser method processIncludes.

/**
     *
     * @param yaml the yaml parser for parsing the include file(s)
     * @param topologyDef the topology definition containing (possibly zero) includes
     * @return The TopologyDef with includes resolved.
     */
private static TopologyDef processIncludes(Yaml yaml, TopologyDef topologyDef, String propsFile, boolean envSub) throws IOException {
    //TODO support multiple levels of includes
    if (topologyDef.getIncludes() != null) {
        for (IncludeDef include : topologyDef.getIncludes()) {
            TopologyDef includeTopologyDef = null;
            if (include.isResource()) {
                LOG.info("Loading includes from resource: {}", include.getFile());
                includeTopologyDef = parseResource(include.getFile(), true, false, propsFile, envSub);
            } else {
                LOG.info("Loading includes from file: {}", include.getFile());
                includeTopologyDef = parseFile(include.getFile(), true, false, propsFile, envSub);
            }
            // if overrides are disabled, we won't replace anything that already exists
            boolean override = include.isOverride();
            // name
            if (includeTopologyDef.getName() != null) {
                topologyDef.setName(includeTopologyDef.getName(), override);
            }
            // config
            if (includeTopologyDef.getConfig() != null) {
                //TODO move this logic to the model class
                Map<String, Object> config = topologyDef.getConfig();
                Map<String, Object> includeConfig = includeTopologyDef.getConfig();
                if (override) {
                    config.putAll(includeTopologyDef.getConfig());
                } else {
                    for (String key : includeConfig.keySet()) {
                        if (config.containsKey(key)) {
                            LOG.warn("Ignoring attempt to set topology config property '{}' with override == false", key);
                        } else {
                            config.put(key, includeConfig.get(key));
                        }
                    }
                }
            }
            //component overrides
            if (includeTopologyDef.getComponents() != null) {
                topologyDef.addAllComponents(includeTopologyDef.getComponents(), override);
            }
            //bolt overrides
            if (includeTopologyDef.getBolts() != null) {
                topologyDef.addAllBolts(includeTopologyDef.getBolts(), override);
            }
            //spout overrides
            if (includeTopologyDef.getSpouts() != null) {
                topologyDef.addAllSpouts(includeTopologyDef.getSpouts(), override);
            }
            //TODO streams should be uniquely identifiable
            if (includeTopologyDef.getStreams() != null) {
                topologyDef.addAllStreams(includeTopologyDef.getStreams(), override);
            }
        }
    // end include processing
    }
    return topologyDef;
}

Example 14 with TopologyDef

use of org.apache.storm.flux.model.TopologyDef in project storm by apache.

the class FluxParser method parseResource.

public static TopologyDef parseResource(String resource, boolean dumpYaml, boolean processIncludes, String propertiesFile, boolean envSub) throws IOException {
    InputStream in = FluxParser.class.getResourceAsStream(resource);
    TopologyDef topology = parseInputStream(in, dumpYaml, processIncludes, propertiesFile, envSub);
    in.close();
    return topology;
}

Example 15 with TopologyDef

use of org.apache.storm.flux.model.TopologyDef in project storm by apache.

the class FluxParser method loadYaml.

private static TopologyDef loadYaml(Yaml yaml, InputStream in, String propsFile, boolean envSubstitution) throws IOException {
    ByteArrayOutputStream bos = new ByteArrayOutputStream();
    LOG.info("loading YAML from input stream...");
    int b = -1;
    while ((b = in.read()) != -1) {
        bos.write(b);
    }
    // TODO substitution implementation is not exactly efficient or kind to memory...
    String str = bos.toString();
    // properties file substitution
    if (propsFile != null) {
        LOG.info("Performing property substitution.");
        InputStream propsIn = new FileInputStream(propsFile);
        Properties props = new Properties();
        props.load(propsIn);
        for (Object key : props.keySet()) {
            str = str.replace("${" + key + "}", props.getProperty((String) key));
        }
    } else {
        LOG.info("Not performing property substitution.");
    }
    // environment variable substitution
    if (envSubstitution) {
        LOG.info("Performing environment variable substitution...");
        Map<String, String> envs = System.getenv();
        for (String key : envs.keySet()) {
            str = str.replace("${ENV-" + key + "}", envs.get(key));
        }
    } else {
        LOG.info("Not performing environment variable substitution.");
    }
    return (TopologyDef) yaml.load(str);
}