Examples with StormTopology org.apache.storm.generated.StormTopology used on opensource projects

Example 66 with StormTopology

use of org.apache.storm.generated.StormTopology in project storm by apache.

the class StreamBuilderTest method testRepartition.

@Test
public void testRepartition() throws Exception {
    Stream<String> stream = streamBuilder.newStream(newSpout(Utils.DEFAULT_STREAM_ID), new ValueMapper<>(0));
    stream.repartition(3).filter(x -> true).repartition(2).filter(x -> true).aggregate(new Count<>());
    StormTopology topology = streamBuilder.build();
    assertEquals(1, topology.get_spouts_size());
    SpoutSpec spout = topology.get_spouts().get("spout1");
    assertEquals(4, topology.get_bolts_size());
    Bolt bolt1 = topology.get_bolts().get("bolt1");
    Bolt bolt2 = topology.get_bolts().get("bolt2");
    Bolt bolt3 = topology.get_bolts().get("bolt3");
    Bolt bolt4 = topology.get_bolts().get("bolt4");
    assertEquals(1, spout.get_common().get_parallelism_hint());
    assertEquals(1, bolt1.get_common().get_parallelism_hint());
    assertEquals(3, bolt2.get_common().get_parallelism_hint());
    assertEquals(2, bolt3.get_common().get_parallelism_hint());
    assertEquals(2, bolt4.get_common().get_parallelism_hint());
}

Example 67 with StormTopology

use of org.apache.storm.generated.StormTopology in project storm by apache.

the class ExecutorTransferMultiThreadingTest method setup.

@Before
public void setup() throws NoSuchFieldException {
    topoConf = Utils.readStormConfig();
    String topologyId = "multi-threaded-topo-test";
    StormTopology stormTopology = createStormTopology();
    WorkerTopologyContext workerTopologyContext = mock(WorkerTopologyContext.class);
    when(workerTopologyContext.getRawTopology()).thenReturn(stormTopology);
    when(workerTopologyContext.getComponentId(selfTaskId)).thenReturn(sourceComp);
    when(workerTopologyContext.getComponentId(remoteTaskId)).thenReturn(destComp);
    workerState = mock(WorkerState.class);
    when(workerState.getWorkerTopologyContext()).thenReturn(workerTopologyContext);
    Map<Integer, JCQueue> receiveQMap = new HashMap<>();
    // local recvQ is not important in this test; simple mock it
    receiveQMap.put(selfTaskId, mock(JCQueue.class));
    when(workerState.getLocalReceiveQueues()).thenReturn(receiveQMap);
    when(workerState.getTopologyId()).thenReturn(topologyId);
    when(workerState.getPort()).thenReturn(6701);
    when(workerState.getMetricRegistry()).thenReturn(new StormMetricRegistry());
    when(workerState.tryTransferRemote(any(), any(), any())).thenCallRealMethod();
    // the actual worker transfer queue to be used in this test
    // taskId for worker transfer queue should be -1.
    // But there is already one worker transfer queue initialized by WorkerTransfer class (taskId=-1).
    // However the taskId is only used for metrics and it is not important here. Making it -100 to avoid collision.
    transferQueue = new JCQueue("worker-transfer-queue", "worker-transfer-queue", 1024, 0, 1, new WaitStrategyPark(100), workerState.getTopologyId(), Constants.SYSTEM_COMPONENT_ID, Collections.singletonList(-100), workerState.getPort(), workerState.getMetricRegistry());
    // Replace the transferQueue inside WorkerTransfer (inside WorkerState) with the customized transferQueue to be used in this test
    WorkerTransfer workerTransfer = new WorkerTransfer(workerState, topoConf, 2);
    FieldSetter.setField(workerTransfer, workerTransfer.getClass().getDeclaredField("transferQueue"), transferQueue);
    FieldSetter.setField(workerState, workerState.getClass().getDeclaredField("workerTransfer"), workerTransfer);
    generalTopologyContext = mock(GeneralTopologyContext.class);
}

Example 68 with StormTopology

use of org.apache.storm.generated.StormTopology in project storm by apache.

the class MessagingTest method testLocalTransport.

@Test
public void testLocalTransport() throws Exception {
    Config topoConf = new Config();
    topoConf.put(Config.TOPOLOGY_WORKERS, 2);
    topoConf.put(Config.STORM_MESSAGING_TRANSPORT, "org.apache.storm.messaging.netty.Context");
    try (ILocalCluster cluster = new LocalCluster.Builder().withSimulatedTime().withSupervisors(1).withPortsPerSupervisor(2).withDaemonConf(topoConf).build()) {
        TopologyBuilder builder = new TopologyBuilder();
        builder.setSpout("1", new TestWordSpout(true), 2);
        builder.setBolt("2", new TestGlobalCount(), 6).shuffleGrouping("1");
        StormTopology stormTopology = builder.createTopology();
        List<FixedTuple> fixedTuples = new ArrayList<>();
        for (int i = 0; i < 12; i++) {
            fixedTuples.add(new FixedTuple(Collections.singletonList("a")));
            fixedTuples.add(new FixedTuple(Collections.singletonList("b")));
        }
        Map<String, List<FixedTuple>> data = new HashMap<>();
        data.put("1", fixedTuples);
        MockedSources mockedSources = new MockedSources(data);
        CompleteTopologyParam completeTopologyParam = new CompleteTopologyParam();
        completeTopologyParam.setMockedSources(mockedSources);
        Map<String, List<FixedTuple>> results = Testing.completeTopology(cluster, stormTopology, completeTopologyParam);
        Assert.assertEquals(6 * 4, Testing.readTuples(results, "2").size());
    }
}

Example 69 with StormTopology

use of org.apache.storm.generated.StormTopology in project storm by apache.

the class TestRebalance method testRebalanceTopologyResourcesAndConfigs.

@Test
public void testRebalanceTopologyResourcesAndConfigs() throws Exception {
    LOG.info("Starting local cluster...");
    Config conf = new Config();
    conf.put(DaemonConfig.STORM_SCHEDULER, ResourceAwareScheduler.class.getName());
    conf.put(DaemonConfig.RESOURCE_AWARE_SCHEDULER_PRIORITY_STRATEGY, DefaultSchedulingPriorityStrategy.class.getName());
    conf.put(Config.TOPOLOGY_SCHEDULER_STRATEGY, getDefaultResourceAwareStrategyClass().getName());
    conf.put(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT, 10.0);
    conf.put(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB, 10.0);
    conf.put(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB, 100.0);
    conf.put(Config.TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB, Double.MAX_VALUE);
    Map<String, Double> resourcesMap = new HashMap();
    resourcesMap.put("gpu.count", 5.0);
    conf.put(Config.TOPOLOGY_COMPONENT_RESOURCES_MAP, resourcesMap);
    try (ILocalCluster cluster = new LocalCluster.Builder().withDaemonConf(conf).build()) {
        TopologyBuilder builder = new TopologyBuilder();
        SpoutDeclarer s1 = builder.setSpout("spout-1", new TestUtilsForResourceAwareScheduler.TestSpout(), 2);
        BoltDeclarer b1 = builder.setBolt("bolt-1", new TestUtilsForResourceAwareScheduler.TestBolt(), 2).shuffleGrouping("spout-1");
        BoltDeclarer b2 = builder.setBolt("bolt-2", new TestUtilsForResourceAwareScheduler.TestBolt(), 2).shuffleGrouping("bolt-1");
        StormTopology stormTopology = builder.createTopology();
        LOG.info("submitting topologies....");
        String topoName = "topo1";
        cluster.submitTopology(topoName, new HashMap<>(), stormTopology);
        waitTopologyScheduled(topoName, cluster, 20);
        RebalanceOptions opts = new RebalanceOptions();
        Map<String, Map<String, Double>> resources = new HashMap<String, Map<String, Double>>();
        resources.put("spout-1", new HashMap<String, Double>());
        resources.get("spout-1").put(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB, 120.0);
        resources.get("spout-1").put(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT, 25.0);
        resources.get("spout-1").put("gpu.count", 5.0);
        opts.set_topology_resources_overrides(resources);
        opts.set_wait_secs(0);
        JSONObject jsonObject = new JSONObject();
        jsonObject.put(Config.TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB, 768.0);
        opts.set_topology_conf_overrides(jsonObject.toJSONString());
        LOG.info("rebalancing....");
        cluster.rebalance("topo1", opts);
        waitTopologyScheduled(topoName, cluster, 10);
        boolean topologyUpdated = false;
        JSONParser parser = new JSONParser();
        for (int i = 0; i < 5; i++) {
            Utils.sleep(SLEEP_TIME_BETWEEN_RETRY);
            String confRaw = cluster.getTopologyConf(topoNameToId(topoName, cluster));
            JSONObject readConf = (JSONObject) parser.parse(confRaw);
            if (768.0 == (double) readConf.get(Config.TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB)) {
                topologyUpdated = true;
                break;
            }
        }
        StormTopology readStormTopology = cluster.getTopology(topoNameToId(topoName, cluster));
        String componentConfRaw = readStormTopology.get_spouts().get("spout-1").get_common().get_json_conf();
        JSONObject readTopologyConf = (JSONObject) parser.parse(componentConfRaw);
        Map<String, Double> componentResources = (Map<String, Double>) readTopologyConf.get(Config.TOPOLOGY_COMPONENT_RESOURCES_MAP);
        assertTrue("Topology has been updated", topologyUpdated);
        assertEquals("Updated CPU correct", 25.0, componentResources.get(Constants.COMMON_CPU_RESOURCE_NAME), 0.001);
        assertEquals("Updated Memory correct", 120.0, componentResources.get(Constants.COMMON_ONHEAP_MEMORY_RESOURCE_NAME), 0.001);
        assertEquals("Updated Generic resource correct", 5.0, componentResources.get("gpu.count"), 0.001);
    }
}

Example 70 with StormTopology

use of org.apache.storm.generated.StormTopology in project storm by apache.

the class BasicContainerTest method testLaunchStorm0version.

@Test
public void testLaunchStorm0version() throws Exception {
    final String topoId = "test_topology_storm_0.x";
    final int supervisorPort = 6628;
    final int port = 8080;
    final String stormHome = ContainerTest.asAbsPath("tmp", "storm-home");
    final String stormLogDir = ContainerTest.asFile(".", "target").getCanonicalPath();
    final String workerId = "worker-id";
    final String stormLocal = ContainerTest.asAbsPath("tmp", "storm-local");
    final String distRoot = ContainerTest.asAbsPath(stormLocal, "supervisor", "stormdist", topoId);
    final File stormcode = new File(distRoot, "stormcode.ser");
    final File stormjar = new File(distRoot, "stormjar.jar");
    final String log4jdir = ContainerTest.asAbsPath(stormHome, "conf");
    final String workerConf = ContainerTest.asAbsPath(log4jdir, "worker.xml");
    final String workerRoot = ContainerTest.asAbsPath(stormLocal, "workers", workerId);
    final String workerTmpDir = ContainerTest.asAbsPath(workerRoot, "tmp");
    final StormTopology st = new StormTopology();
    st.set_spouts(new HashMap<>());
    st.set_bolts(new HashMap<>());
    st.set_state_spouts(new HashMap<>());
    // minimum 0.x version of supporting STORM-2448 would be 0.10.3
    st.set_storm_version("0.10.3");
    byte[] serializedState = Utils.gzip(Utils.thriftSerialize(st));
    final Map<String, Object> superConf = new HashMap<>();
    superConf.put(Config.STORM_LOCAL_DIR, stormLocal);
    superConf.put(Config.STORM_WORKERS_ARTIFACTS_DIR, stormLocal);
    superConf.put(DaemonConfig.STORM_LOG4J2_CONF_DIR, log4jdir);
    superConf.put(Config.WORKER_CHILDOPTS, " -Dtesting=true");
    LocalAssignment la = new LocalAssignment();
    la.set_topology_id(topoId);
    AdvancedFSOps ops = mock(AdvancedFSOps.class);
    when(ops.doRequiredTopoFilesExist(superConf, topoId)).thenReturn(true);
    when(ops.slurp(stormcode)).thenReturn(serializedState);
    LocalState ls = mock(LocalState.class);
    MockResourceIsolationManager iso = new MockResourceIsolationManager();
    checkpoint(() -> {
        MockBasicContainer mc = new MockBasicContainer(ContainerType.LAUNCH, superConf, "SUPERVISOR", supervisorPort, port, la, iso, ls, workerId, new StormMetricsRegistry(), new HashMap<>(), ops, "profile");
        mc.launch();
        assertEquals(1, iso.workerCmds.size());
        CommandRun cmd = iso.workerCmds.get(0);
        iso.workerCmds.clear();
        assertListEquals(Arrays.asList("java", "-cp", "FRAMEWORK_CP:" + stormjar.getAbsolutePath(), "-Dlogging.sensitivity=S3", "-Dlogfile.name=worker.log", "-Dstorm.home=" + stormHome, "-Dworkers.artifacts=" + stormLocal, "-Dstorm.id=" + topoId, "-Dworker.id=" + workerId, "-Dworker.port=" + port, "-Dstorm.log.dir=" + stormLogDir, "-DLog4jContextSelector=org.apache.logging.log4j.core.selector.BasicContextSelector", "-Dstorm.local.dir=" + stormLocal, "-Dworker.memory_limit_mb=768", "-Dlog4j.configurationFile=" + workerConf, "backtype.storm.LogWriter", "java", "-server", "-Dlogging.sensitivity=S3", "-Dlogfile.name=worker.log", "-Dstorm.home=" + stormHome, "-Dworkers.artifacts=" + stormLocal, "-Dstorm.id=" + topoId, "-Dworker.id=" + workerId, "-Dworker.port=" + port, "-Dstorm.log.dir=" + stormLogDir, "-DLog4jContextSelector=org.apache.logging.log4j.core.selector.BasicContextSelector", "-Dstorm.local.dir=" + stormLocal, "-Dworker.memory_limit_mb=768", "-Dlog4j.configurationFile=" + workerConf, "-Dtesting=true", "-Djava.library.path=JLP", "-Dstorm.conf.file=", "-Dstorm.options=", "-Djava.io.tmpdir=" + workerTmpDir, "-cp", "FRAMEWORK_CP:" + stormjar.getAbsolutePath(), "backtype.storm.daemon.worker", topoId, "SUPERVISOR", String.valueOf(port), workerId), cmd.cmd);
        assertEquals(new File(workerRoot), cmd.pwd);
    }, ConfigUtils.STORM_HOME, stormHome, "storm.log.dir", stormLogDir);
}