Examples with Config com.twitter.heron.api.Config used on opensource projects

Example 41 with Config

use of com.twitter.heron.api.Config in project heron by twitter.

the class AbstractTestTopology method submit.

public final void submit() throws AlreadyAliveException, InvalidTopologyException {
    TestTopologyBuilder builder = new TestTopologyBuilder(httpServerResultsUrl, httpServerStateUrl, stateUpdateToken, spoutWrapperType);
    Config conf = buildConfig(new BasicConfig());
    HeronSubmitter.submitTopology(topologyName, conf, buildTopology(builder).createTopology());
}

Example 42 with Config

use of com.twitter.heron.api.Config in project heron by twitter.

the class UnitTestHelper method setTopology.

private static void setTopology(PhysicalPlans.PhysicalPlan.Builder pPlan, boolean ackEnabled, int messageTimeout, TopologyAPI.TopologyState topologyState) {
    TopologyBuilder topologyBuilder = new TopologyBuilder();
    topologyBuilder.setSpout("test-spout", new TestSpout(), 1);
    // Here we need case switch to corresponding grouping
    topologyBuilder.setBolt("test-bolt", new TestBolt(), 1).shuffleGrouping("test-spout");
    Config conf = new Config();
    conf.setTeamEmail("streaming-compute@twitter.com");
    conf.setTeamName("stream-computing");
    conf.setTopologyProjectName("heron-integration-test");
    conf.setNumStmgrs(1);
    conf.setMaxSpoutPending(100);
    if (ackEnabled) {
        conf.setEnableAcking(true);
    } else {
        conf.setEnableAcking(false);
    }
    if (messageTimeout != -1) {
        conf.setMessageTimeoutSecs(messageTimeout);
        conf.put("topology.enable.message.timeouts", "true");
    }
    TopologyAPI.Topology fTopology = topologyBuilder.createTopology().setName("topology-name").setConfig(conf).setState(topologyState).getTopology();
    pPlan.setTopology(fTopology);
}

Example 43 with Config

use of com.twitter.heron.api.Config in project heron by twitter.

the class CustomGroupingTest method constructPhysicalPlan.

private PhysicalPlans.PhysicalPlan constructPhysicalPlan(MyCustomGrouping myCustomGrouping) {
    PhysicalPlans.PhysicalPlan.Builder pPlan = PhysicalPlans.PhysicalPlan.newBuilder();
    // Set topology protobuf
    TopologyBuilder topologyBuilder = new TopologyBuilder();
    topologyBuilder.setSpout("test-spout", new TestSpout(), 1);
    // Here we need case switch to corresponding grouping
    topologyBuilder.setBolt("test-bolt", new TestBolt(), 1).customGrouping("test-spout", myCustomGrouping);
    Config conf = new Config();
    conf.setTeamEmail("streaming-compute@twitter.com");
    conf.setTeamName("stream-computing");
    conf.setTopologyProjectName("heron-integration-test");
    conf.setNumStmgrs(1);
    conf.setMaxSpoutPending(100);
    conf.setEnableAcking(false);
    TopologyAPI.Topology fTopology = topologyBuilder.createTopology().setName("topology-name").setConfig(conf).setState(TopologyAPI.TopologyState.RUNNING).getTopology();
    pPlan.setTopology(fTopology);
    // Set instances
    // Construct the spoutInstance
    PhysicalPlans.InstanceInfo.Builder spoutInstanceInfo = PhysicalPlans.InstanceInfo.newBuilder();
    spoutInstanceInfo.setComponentName("test-spout");
    spoutInstanceInfo.setTaskId(0);
    spoutInstanceInfo.setComponentIndex(0);
    PhysicalPlans.Instance.Builder spoutInstance = PhysicalPlans.Instance.newBuilder();
    spoutInstance.setInstanceId("spout-id");
    spoutInstance.setStmgrId("stream-manager-id");
    spoutInstance.setInfo(spoutInstanceInfo);
    // Construct the boltInstanceInfo
    PhysicalPlans.InstanceInfo.Builder boltInstanceInfo = PhysicalPlans.InstanceInfo.newBuilder();
    boltInstanceInfo.setComponentName("test-bolt");
    boltInstanceInfo.setTaskId(1);
    boltInstanceInfo.setComponentIndex(0);
    PhysicalPlans.Instance.Builder boltInstance = PhysicalPlans.Instance.newBuilder();
    boltInstance.setInstanceId("bolt-id");
    boltInstance.setStmgrId("stream-manager-id");
    boltInstance.setInfo(boltInstanceInfo);
    pPlan.addInstances(spoutInstance);
    pPlan.addInstances(boltInstance);
    // Set stream mgr
    PhysicalPlans.StMgr.Builder stmgr = PhysicalPlans.StMgr.newBuilder();
    stmgr.setId("stream-manager-id");
    stmgr.setHostName("127.0.0.1");
    stmgr.setDataPort(8888);
    stmgr.setLocalEndpoint("endpoint");
    pPlan.addStmgrs(stmgr);
    return pPlan.build();
}

Example 44 with Config

use of com.twitter.heron.api.Config in project heron by twitter.

the class TopologyUtilsTest method testGetComponentParallelism.

@Test
public void testGetComponentParallelism() {
    int componentParallelism = 4;
    Config topologyConfig = new Config();
    Map<String, Integer> spouts = new HashMap<>();
    spouts.put("spout", componentParallelism);
    Map<String, Integer> bolts = new HashMap<>();
    bolts.put("bolt", componentParallelism);
    TopologyAPI.Topology topology = TopologyTests.createTopology("testTopology", topologyConfig, spouts, bolts);
    Map<String, Integer> componentParallelismMap = TopologyUtils.getComponentParallelism(topology);
    Assert.assertEquals(componentParallelism, componentParallelismMap.get("spout").intValue());
    Assert.assertEquals(componentParallelism, componentParallelismMap.get("bolt").intValue());
}

Example 45 with Config

use of com.twitter.heron.api.Config in project heron by twitter.

the class TopologyUtilsTest method testGetComponentRamMapAllRamSpecified.

@Test
public void testGetComponentRamMapAllRamSpecified() {
    int componentParallelism = 2;
    Config topologyConfig = new Config();
    Map<String, Integer> spouts = new HashMap<>();
    spouts.put("spout", componentParallelism);
    Map<String, Integer> bolts = new HashMap<>();
    bolts.put("bolt", componentParallelism);
    ByteAmount boltRam = ByteAmount.fromGigabytes(1);
    ByteAmount spoutRam = ByteAmount.fromGigabytes(2);
    topologyConfig.setComponentRam("spout", spoutRam);
    topologyConfig.setComponentRam("bolt", boltRam);
    // sort the component ram map
    Map<String, ByteAmount> ramMap = new TreeMap<>(TopologyUtils.getComponentRamMapConfig(TopologyTests.createTopology("test", topologyConfig, spouts, bolts)));
    Assert.assertArrayEquals(new String[] { "bolt", "spout" }, ramMap.keySet().toArray());
    Assert.assertArrayEquals(new ByteAmount[] { boltRam, spoutRam }, ramMap.values().toArray());
}