Examples with TopologyBuilder org.apache.storm.topology.TopologyBuilder used on opensource projects

Example 81 with TopologyBuilder

use of org.apache.storm.topology.TopologyBuilder in project storm by apache.

the class TestHdfsBolt method generateTestTuple.

private Tuple generateTestTuple(Object id, Object msg, Object city, Object state) {
    TopologyBuilder builder = new TopologyBuilder();
    GeneralTopologyContext topologyContext = new GeneralTopologyContext(builder.createTopology(), new Config(), new HashMap<>(), new HashMap<>(), new HashMap<>(), "") {

        @Override
        public Fields getComponentOutputFields(String componentId, String streamId) {
            return new Fields("id", "msg", "city", "state");
        }
    };
    return new TupleImpl(topologyContext, new Values(id, msg, city, state), topologyContext.getComponentId(1), 1, "");
}

Example 82 with TopologyBuilder

use of org.apache.storm.topology.TopologyBuilder in project storm by apache.

the class EventCount method buildTopology.

protected StormTopology buildTopology(EventHubSpout eventHubSpout) {
    TopologyBuilder topologyBuilder = new TopologyBuilder();
    topologyBuilder.setSpout("EventHubsSpout", eventHubSpout, spoutConfig.getPartitionCount()).setNumTasks(spoutConfig.getPartitionCount());
    topologyBuilder.setBolt("PartialCountBolt", new PartialCountBolt(), spoutConfig.getPartitionCount()).localOrShuffleGrouping("EventHubsSpout").setNumTasks(spoutConfig.getPartitionCount());
    topologyBuilder.setBolt("GlobalCountBolt", new GlobalCountBolt(), 1).globalGrouping("PartialCountBolt").setNumTasks(1);
    return topologyBuilder.createTopology();
}

Example 83 with TopologyBuilder

use of org.apache.storm.topology.TopologyBuilder in project storm by apache.

the class EsTestUtil method generateTestTuple.

public static Tuple generateTestTuple(String source, String index, String type, String id) {
    TopologyBuilder builder = new TopologyBuilder();
    GeneralTopologyContext topologyContext = new GeneralTopologyContext(builder.createTopology(), new Config(), new HashMap<>(), new HashMap<>(), new HashMap<>(), "") {

        @Override
        public Fields getComponentOutputFields(String componentId, String streamId) {
            return new Fields("source", "index", "type", "id");
        }
    };
    return new TupleImpl(topologyContext, new Values(source, index, type, id), source, 1, "");
}

Example 84 with TopologyBuilder

use of org.apache.storm.topology.TopologyBuilder in project storm by apache.

the class EventHubLoop method buildTopology.

@Override
protected StormTopology buildTopology(EventHubSpout eventHubSpout) {
    TopologyBuilder topologyBuilder = new TopologyBuilder();
    topologyBuilder.setSpout("EventHubsSpout", eventHubSpout, spoutConfig.getPartitionCount()).setNumTasks(spoutConfig.getPartitionCount());
    EventHubBoltConfig boltConfig = new EventHubBoltConfig(spoutConfig.getConnectionString(), spoutConfig.getEntityPath(), true);
    EventHubBolt eventHubBolt = new EventHubBolt(boltConfig);
    int boltTasks = spoutConfig.getPartitionCount();
    topologyBuilder.setBolt("EventHubsBolt", eventHubBolt, boltTasks).localOrShuffleGrouping("EventHubsSpout").setNumTasks(boltTasks);
    return topologyBuilder.createTopology();
}

Example 85 with TopologyBuilder

use of org.apache.storm.topology.TopologyBuilder in project storm by apache.

the class TestResourceAwareScheduler method testHeterogeneousCluster.

public void testHeterogeneousCluster(Config topologyConf, String strategyName) {
    LOG.info("\n\n\t\ttestHeterogeneousCluster");
    INimbus iNimbus = new INimbusTest();
    // strong supervisor node
    Map<String, Double> resourceMap1 = new HashMap<>();
    resourceMap1.put(Config.SUPERVISOR_CPU_CAPACITY, 800.0);
    resourceMap1.put(Config.SUPERVISOR_MEMORY_CAPACITY_MB, 4096.0);
    // weak supervisor node
    Map<String, Double> resourceMap2 = new HashMap<>();
    resourceMap2.put(Config.SUPERVISOR_CPU_CAPACITY, 200.0);
    resourceMap2.put(Config.SUPERVISOR_MEMORY_CAPACITY_MB, 1024.0);
    resourceMap1 = NormalizedResources.RESOURCE_NAME_NORMALIZER.normalizedResourceMap(resourceMap1);
    resourceMap2 = NormalizedResources.RESOURCE_NAME_NORMALIZER.normalizedResourceMap(resourceMap2);
    Map<String, SupervisorDetails> supMap = new HashMap<>();
    for (int i = 0; i < 2; i++) {
        List<Number> ports = new LinkedList<>();
        for (int j = 0; j < 4; j++) {
            ports.add(j);
        }
        SupervisorDetails sup = new SupervisorDetails("r00s00" + i, "host-" + i, null, ports, i == 0 ? resourceMap1 : resourceMap2);
        supMap.put(sup.getId(), sup);
    }
    LOG.info("SUPERVISORS = {}", supMap);
    // topo1 has one single huge task that can not be handled by the small-super
    TopologyBuilder builder1 = new TopologyBuilder();
    builder1.setSpout("wordSpout1", new TestWordSpout(), 1).setCPULoad(300.0).setMemoryLoad(2000.0, 48.0);
    StormTopology stormTopology1 = builder1.createTopology();
    Config config1 = new Config();
    config1.putAll(topologyConf);
    Map<ExecutorDetails, String> executorMap1 = genExecsAndComps(stormTopology1);
    TopologyDetails topology1 = new TopologyDetails("topology1", config1, stormTopology1, 1, executorMap1, 0, "user");
    // topo2 has 4 large tasks
    TopologyBuilder builder2 = new TopologyBuilder();
    builder2.setSpout("wordSpout2", new TestWordSpout(), 4).setCPULoad(100.0).setMemoryLoad(500.0, 12.0);
    StormTopology stormTopology2 = builder2.createTopology();
    Config config2 = new Config();
    config2.putAll(topologyConf);
    Map<ExecutorDetails, String> executorMap2 = genExecsAndComps(stormTopology2);
    TopologyDetails topology2 = new TopologyDetails("topology2", config2, stormTopology2, 1, executorMap2, 0, "user");
    // topo3 has 4 large tasks
    TopologyBuilder builder3 = new TopologyBuilder();
    builder3.setSpout("wordSpout3", new TestWordSpout(), 4).setCPULoad(20.0).setMemoryLoad(200.0, 56.0);
    StormTopology stormTopology3 = builder3.createTopology();
    Config config3 = new Config();
    config3.putAll(topologyConf);
    Map<ExecutorDetails, String> executorMap3 = genExecsAndComps(stormTopology3);
    TopologyDetails topology3 = new TopologyDetails("topology3", config3, stormTopology3, 1, executorMap3, 0, "user");
    // topo4 has 12 small tasks, whose mem usage does not exactly divide a node's mem capacity
    TopologyBuilder builder4 = new TopologyBuilder();
    builder4.setSpout("wordSpout4", new TestWordSpout(), 12).setCPULoad(30.0).setMemoryLoad(100.0, 0.0);
    StormTopology stormTopology4 = builder4.createTopology();
    Config config4 = new Config();
    config4.putAll(topologyConf);
    Map<ExecutorDetails, String> executorMap4 = genExecsAndComps(stormTopology4);
    TopologyDetails topology4 = new TopologyDetails("topology4", config4, stormTopology4, 1, executorMap4, 0, "user");
    // topo5 has 40 small tasks, it should be able to exactly use up both the cpu and mem in the cluster
    TopologyBuilder builder5 = new TopologyBuilder();
    builder5.setSpout("wordSpout5", new TestWordSpout(), 40).setCPULoad(25.0).setMemoryLoad(100.0, 28.0);
    StormTopology stormTopology5 = builder5.createTopology();
    Config config5 = new Config();
    config5.putAll(topologyConf);
    Map<ExecutorDetails, String> executorMap5 = genExecsAndComps(stormTopology5);
    TopologyDetails topology5 = new TopologyDetails("topology5", config5, stormTopology5, 1, executorMap5, 0, "user");
    // Test1: Launch topo 1-3 together, it should be able to use up either mem or cpu resource due to exact division
    ResourceAwareScheduler rs = new ResourceAwareScheduler();
    LOG.info("\n\n\t\tScheduling topologies 1, 2 and 3");
    Topologies topologies = new Topologies(topology1, topology2, topology3);
    Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config1);
    rs.prepare(config1, new StormMetricsRegistry());
    Map<SupervisorDetails, Double> superToCpu = null;
    Map<SupervisorDetails, Double> superToMem = null;
    try {
        rs.schedule(topologies, cluster);
        assertFalse(cluster.needsSchedulingRas(topology1));
        assertFalse(cluster.needsSchedulingRas(topology2));
        assertFalse(cluster.needsSchedulingRas(topology3));
        String expectedMsgPrefix = "Running - Fully Scheduled by " + strategyName;
        assertTrue(cluster.getStatusMap().get(topology1.getId()).startsWith(expectedMsgPrefix));
        assertTrue(cluster.getStatusMap().get(topology2.getId()).startsWith(expectedMsgPrefix));
        assertTrue(cluster.getStatusMap().get(topology3.getId()).startsWith(expectedMsgPrefix));
        superToCpu = getSupervisorToCpuUsage(cluster, topologies);
        superToMem = getSupervisorToMemoryUsage(cluster, topologies);
        final Double EPSILON = 0.0001;
        for (SupervisorDetails supervisor : supMap.values()) {
            Double cpuAvailable = supervisor.getTotalCpu();
            Double memAvailable = supervisor.getTotalMemory();
            Double cpuUsed = superToCpu.get(supervisor);
            Double memUsed = superToMem.get(supervisor);
            assertTrue(supervisor.getId() + " MEM: " + memAvailable + " == " + memUsed + " OR CPU: " + cpuAvailable + " == " + cpuUsed, (Math.abs(memAvailable - memUsed) < EPSILON) || (Math.abs(cpuAvailable - cpuUsed) < EPSILON));
        }
    } finally {
        rs.cleanup();
    }
    // end of Test1
    LOG.warn("\n\n\t\tSwitching to topologies 1, 2 and 4");
    // Test2: Launch topo 1, 2 and 4, they together request a little more mem than available, so one of the 3 topos will not be
    // scheduled
    topologies = new Topologies(topology1, topology2, topology4);
    cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config1);
    rs.prepare(config1, new StormMetricsRegistry());
    try {
        rs.schedule(topologies, cluster);
        int numTopologiesAssigned = 0;
        if (cluster.getStatusMap().get(topology1.getId()).startsWith("Running - Fully Scheduled by " + strategyName)) {
            LOG.info("TOPO 1 scheduled");
            numTopologiesAssigned++;
        }
        if (cluster.getStatusMap().get(topology2.getId()).startsWith("Running - Fully Scheduled by " + strategyName)) {
            LOG.info("TOPO 2 scheduled");
            numTopologiesAssigned++;
        }
        if (cluster.getStatusMap().get(topology4.getId()).startsWith("Running - Fully Scheduled by " + strategyName)) {
            LOG.info("TOPO 3 scheduled");
            numTopologiesAssigned++;
        }
        assertEquals(2, numTopologiesAssigned);
    } finally {
        rs.cleanup();
    }
    // end of Test2
    LOG.info("\n\n\t\tScheduling just topo 5");
    // Test3: "Launch topo5 only, both mem and cpu should be exactly used up"
    topologies = new Topologies(topology5);
    cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config1);
    rs.prepare(config1, new StormMetricsRegistry());
    try {
        rs.schedule(topologies, cluster);
        superToCpu = getSupervisorToCpuUsage(cluster, topologies);
        superToMem = getSupervisorToMemoryUsage(cluster, topologies);
        for (SupervisorDetails supervisor : supMap.values()) {
            Double cpuAvailable = supervisor.getTotalCpu();
            Double memAvailable = supervisor.getTotalMemory();
            Double cpuUsed = superToCpu.get(supervisor);
            Double memUsed = superToMem.get(supervisor);
            assertEquals(cpuAvailable, cpuUsed, 0.0001);
            assertEquals(memAvailable, memUsed, 0.0001);
        }
    } finally {
        rs.cleanup();
    }
// end of Test3
}