Examples with Config org.apache.heron.api.Config used on opensource projects

Example 36 with Config

use of org.apache.heron.api.Config in project heron by twitter.

the class MockPhysicalPlansBuilder method withTopologyConfig.

public MockPhysicalPlansBuilder withTopologyConfig(Config.TopologyReliabilityMode reliabilityMode, int messageTimeout) {
    conf = new Config();
    conf.setTeamEmail("streaming-compute@twitter.com");
    conf.setTeamName("stream-computing");
    conf.setTopologyProjectName("heron-integration-test");
    conf.setNumStmgrs(1);
    conf.setMaxSpoutPending(100);
    conf.setTopologyReliabilityMode(reliabilityMode);
    if (messageTimeout != -1) {
        conf.setMessageTimeoutSecs(messageTimeout);
        conf.put("topology.enable.message.timeouts", "true");
    }
    return this;
}

Example 37 with Config

use of org.apache.heron.api.Config in project heron by twitter.

the class AbstractTestTopology method submit.

public final void submit(Config userConf) throws AlreadyAliveException, InvalidTopologyException {
    TestTopologyBuilder builder = new TestTopologyBuilder(httpServerResultsUrl, httpServerStateUrl, stateUpdateToken, spoutWrapperType);
    Config conf = buildConfig(new BasicConfig());
    if (userConf != null) {
        conf.putAll(userConf);
    }
    HeronSubmitter.submitTopology(topologyName, conf, buildTopology(builder).createTopology());
}

Example 38 with Config

use of org.apache.heron.api.Config in project heron by twitter.

the class Eco method submit.

/**
 * Submit an ECO topology
 *
 * @param fileInputStream  The input stream associated with ECO topology definition file
 * @param propertiesFile  The optional key-value property file for optional property substitution.
 * @param envFilter The optional flag to tell ECO to perform environment variable substitution
 * @throws Exception the exception thrown
 */
public void submit(FileInputStream fileInputStream, FileInputStream propertiesFile, boolean envFilter) throws Exception {
    EcoTopologyDefinition topologyDefinition = ecoParser.parseFromInputStream(fileInputStream, propertiesFile, envFilter);
    String topologyName = topologyDefinition.getName();
    String topologyType = topologyDefinition.getType();
    if ("storm".equals(topologyType)) {
        System.out.println("topology type is Storm");
        org.apache.heron.eco.builder.storm.EcoBuilder ecoBuilder = new org.apache.heron.eco.builder.storm.EcoBuilder(new org.apache.heron.eco.builder.storm.SpoutBuilder(), new BoltBuilder(), new org.apache.heron.eco.builder.storm.StreamBuilder(), new ComponentBuilder(), new ConfigBuilder());
        Config topologyConfig = ecoBuilder.buildConfig(topologyDefinition);
        EcoExecutionContext executionContext = new EcoExecutionContext(topologyDefinition, topologyConfig);
        printTopologyInfo(executionContext);
        ObjectBuilder objectBuilder = new ObjectBuilder();
        objectBuilder.setBuilderUtility(new BuilderUtility());
        org.apache.storm.topology.TopologyBuilder builder = ecoBuilder.buildTopologyBuilder(executionContext, objectBuilder);
        ecoSubmitter.submitStormTopology(topologyName, topologyConfig, builder.createTopology());
    } else if ("heron".equals(topologyType)) {
        System.out.println("topology type is Heron");
        org.apache.heron.eco.builder.heron.EcoBuilder ecoBuilder = new org.apache.heron.eco.builder.heron.EcoBuilder(new org.apache.heron.eco.builder.heron.SpoutBuilder(), new BoltBuilder(), new org.apache.heron.eco.builder.heron.StreamBuilder(), new ComponentBuilder(), new ConfigBuilder());
        Config topologyConfig = ecoBuilder.buildConfig(topologyDefinition);
        EcoExecutionContext executionContext = new EcoExecutionContext(topologyDefinition, topologyConfig);
        printTopologyInfo(executionContext);
        ObjectBuilder objectBuilder = new ObjectBuilder();
        objectBuilder.setBuilderUtility(new BuilderUtility());
        org.apache.heron.api.topology.TopologyBuilder builder = ecoBuilder.buildTopologyBuilder(executionContext, objectBuilder);
        ecoSubmitter.submitHeronTopology(topologyName, topologyConfig, builder.createTopology());
    } else {
        LOG.log(Level.SEVERE, String.format("Unknown topology type \'%s\' for topology %s, not submitted", topologyType, topologyName));
    }
}

Example 39 with Config

use of org.apache.heron.api.Config in project heron by twitter.

the class ConfigBuilder method buildConfig.

/**
 * Build the config for a ECO topology definition
 *
 * @param topologyDefinition - ECO topology definition
 */
public Config buildConfig(EcoTopologyDefinition topologyDefinition) throws IllegalArgumentException {
    Map<String, Object> configMap = topologyDefinition.getConfig();
    Config config = new Config();
    for (Map.Entry<String, Object> entry : configMap.entrySet()) {
        if (entry.getKey().equals(COMPONENT_RESOURCE_MAP)) {
            setComponentLevelResource(config, entry);
        } else if (entry.getKey().equals(COMPONENT_JVM_OPTIONS)) {
            List<Object> objects = (List<Object>) entry.getValue();
            for (Object obj : objects) {
                String objString = obj.toString();
                objString = objString.replace(LEFT_BRACE, WHITESPACE);
                objString = objString.replace(RIGHT_BRACE, WHITESPACE);
                int idIndex = objString.indexOf(ID);
                int optionsIndex = objString.indexOf(OPTIONS);
                String id = getIdValue(objString, idIndex);
                String jvmOptions;
                if (optionsIndex != -1) {
                    int equalsIndex = objString.indexOf(EQUALS, optionsIndex);
                    jvmOptions = objString.substring(equalsIndex + 1, objString.length());
                    jvmOptions = jvmOptions.replace(LEFT_BRACKET, "").replace(RIGHT_BRACKET, "");
                } else {
                    throw new IllegalArgumentException("You must specify the JVM options for your component");
                }
                config.setComponentJvmOptions(id, jvmOptions);
            }
        } else {
            config.put(entry.getKey(), entry.getValue());
        }
    }
    return config;
}

Example 40 with Config

use of org.apache.heron.api.Config in project heron by twitter.

the class StormEcoBuilderTest method testBuild_CustomConfigMap_ReturnsCorrectConfigs.

@Test
public void testBuild_CustomConfigMap_ReturnsCorrectConfigs() throws Exception {
    configMap.put(Config.TOPOLOGY_DEBUG, false);
    final String environment = "dev";
    final int spouts = 3;
    configMap.put(Config.TOPOLOGY_ENVIRONMENT, environment);
    configMap.put(Config.TOPOLOGY_MAX_SPOUT_PENDING, spouts);
    Config config = new Config();
    when(mockConfigBuilder.buildConfig(eq(ecoTopologyDefinition))).thenReturn(config);
    assertThat(subject.buildConfig(ecoTopologyDefinition), sameInstance(config));
    verify(mockConfigBuilder).buildConfig(same(ecoTopologyDefinition));
}