Examples with StormTopology org.apache.storm.generated.StormTopology used on opensource projects

Example 71 with StormTopology

use of org.apache.storm.generated.StormTopology in project storm by apache.

the class BasicContainerTest method testLaunch.

@Test
public void testLaunch() throws Exception {
    final String topoId = "test_topology_current";
    final int supervisorPort = 6628;
    final int port = 8080;
    final String stormHome = ContainerTest.asAbsPath("tmp", "storm-home");
    final String stormLogDir = ContainerTest.asFile(".", "target").getCanonicalPath();
    final String workerId = "worker-id";
    final String stormLocal = ContainerTest.asAbsPath("tmp", "storm-local");
    final String distRoot = ContainerTest.asAbsPath(stormLocal, "supervisor", "stormdist", topoId);
    final File stormcode = new File(distRoot, "stormcode.ser");
    final File stormjar = new File(distRoot, "stormjar.jar");
    final String log4jdir = ContainerTest.asAbsPath(stormHome, "conf");
    final String workerConf = ContainerTest.asAbsPath(log4jdir, "worker.xml");
    final String workerRoot = ContainerTest.asAbsPath(stormLocal, "workers", workerId);
    final String workerTmpDir = ContainerTest.asAbsPath(workerRoot, "tmp");
    final StormTopology st = new StormTopology();
    st.set_spouts(new HashMap<>());
    st.set_bolts(new HashMap<>());
    st.set_state_spouts(new HashMap<>());
    byte[] serializedState = Utils.gzip(Utils.thriftSerialize(st));
    final Map<String, Object> superConf = new HashMap<>();
    superConf.put(Config.STORM_LOCAL_DIR, stormLocal);
    superConf.put(Config.STORM_WORKERS_ARTIFACTS_DIR, stormLocal);
    superConf.put(DaemonConfig.STORM_LOG4J2_CONF_DIR, log4jdir);
    superConf.put(Config.WORKER_CHILDOPTS, " -Dtesting=true");
    LocalAssignment la = new LocalAssignment();
    la.set_topology_id(topoId);
    AdvancedFSOps ops = mock(AdvancedFSOps.class);
    when(ops.doRequiredTopoFilesExist(superConf, topoId)).thenReturn(true);
    when(ops.slurp(stormcode)).thenReturn(serializedState);
    LocalState ls = mock(LocalState.class);
    MockResourceIsolationManager iso = new MockResourceIsolationManager();
    checkpoint(() -> {
        MockBasicContainer mc = new MockBasicContainer(ContainerType.LAUNCH, superConf, "SUPERVISOR", supervisorPort, port, la, iso, ls, workerId, new StormMetricsRegistry(), new HashMap<>(), ops, "profile");
        mc.launch();
        assertEquals(1, iso.workerCmds.size());
        CommandRun cmd = iso.workerCmds.get(0);
        iso.workerCmds.clear();
        assertListEquals(Arrays.asList("java", "-cp", "FRAMEWORK_CP:" + stormjar.getAbsolutePath(), "-Dlogging.sensitivity=S3", "-Dlogfile.name=worker.log", "-Dstorm.home=" + stormHome, "-Dworkers.artifacts=" + stormLocal, "-Dstorm.id=" + topoId, "-Dworker.id=" + workerId, "-Dworker.port=" + port, "-Dstorm.log.dir=" + stormLogDir, "-DLog4jContextSelector=org.apache.logging.log4j.core.selector.BasicContextSelector", "-Dstorm.local.dir=" + stormLocal, "-Dworker.memory_limit_mb=768", "-Dlog4j.configurationFile=" + workerConf, "org.apache.storm.LogWriter", "java", "-server", "-Dlogging.sensitivity=S3", "-Dlogfile.name=worker.log", "-Dstorm.home=" + stormHome, "-Dworkers.artifacts=" + stormLocal, "-Dstorm.id=" + topoId, "-Dworker.id=" + workerId, "-Dworker.port=" + port, "-Dstorm.log.dir=" + stormLogDir, "-DLog4jContextSelector=org.apache.logging.log4j.core.selector.BasicContextSelector", "-Dstorm.local.dir=" + stormLocal, "-Dworker.memory_limit_mb=768", "-Dlog4j.configurationFile=" + workerConf, "-Dtesting=true", "-Djava.library.path=JLP", "-Dstorm.conf.file=", "-Dstorm.options=", "-Djava.io.tmpdir=" + workerTmpDir, "-cp", "FRAMEWORK_CP:" + stormjar.getAbsolutePath(), "org.apache.storm.daemon.worker.Worker", topoId, "SUPERVISOR", String.valueOf(supervisorPort), String.valueOf(port), workerId), cmd.cmd);
        assertEquals(new File(workerRoot), cmd.pwd);
    }, ConfigUtils.STORM_HOME, stormHome, "storm.log.dir", stormLogDir);
}

Example 72 with StormTopology

use of org.apache.storm.generated.StormTopology in project storm by apache.

the class BasicContainerTest method testLaunchStorm1version.

@Test
public void testLaunchStorm1version() throws Exception {
    final String topoId = "test_topology_storm_1.x";
    final int supervisorPort = 6628;
    final int port = 8080;
    final String stormHome = ContainerTest.asAbsPath("tmp", "storm-home");
    final String stormLogDir = ContainerTest.asFile(".", "target").getCanonicalPath();
    final String workerId = "worker-id";
    final String stormLocal = ContainerTest.asAbsPath("tmp", "storm-local");
    final String distRoot = ContainerTest.asAbsPath(stormLocal, "supervisor", "stormdist", topoId);
    final File stormcode = new File(distRoot, "stormcode.ser");
    final File stormjar = new File(distRoot, "stormjar.jar");
    final String log4jdir = ContainerTest.asAbsPath(stormHome, "conf");
    final String workerConf = ContainerTest.asAbsPath(log4jdir, "worker.xml");
    final String workerRoot = ContainerTest.asAbsPath(stormLocal, "workers", workerId);
    final String workerTmpDir = ContainerTest.asAbsPath(workerRoot, "tmp");
    final StormTopology st = new StormTopology();
    st.set_spouts(new HashMap<>());
    st.set_bolts(new HashMap<>());
    st.set_state_spouts(new HashMap<>());
    // minimum 1.x version of supporting STORM-2448 would be 1.0.4
    st.set_storm_version("1.0.4");
    byte[] serializedState = Utils.gzip(Utils.thriftSerialize(st));
    final Map<String, Object> superConf = new HashMap<>();
    superConf.put(Config.STORM_LOCAL_DIR, stormLocal);
    superConf.put(Config.STORM_WORKERS_ARTIFACTS_DIR, stormLocal);
    superConf.put(DaemonConfig.STORM_LOG4J2_CONF_DIR, log4jdir);
    superConf.put(Config.WORKER_CHILDOPTS, " -Dtesting=true");
    LocalAssignment la = new LocalAssignment();
    la.set_topology_id(topoId);
    AdvancedFSOps ops = mock(AdvancedFSOps.class);
    when(ops.doRequiredTopoFilesExist(superConf, topoId)).thenReturn(true);
    when(ops.slurp(stormcode)).thenReturn(serializedState);
    LocalState ls = mock(LocalState.class);
    MockResourceIsolationManager iso = new MockResourceIsolationManager();
    checkpoint(() -> {
        MockBasicContainer mc = new MockBasicContainer(ContainerType.LAUNCH, superConf, "SUPERVISOR", supervisorPort, port, la, iso, ls, workerId, new StormMetricsRegistry(), new HashMap<>(), ops, "profile");
        mc.launch();
        assertEquals(1, iso.workerCmds.size());
        CommandRun cmd = iso.workerCmds.get(0);
        iso.workerCmds.clear();
        assertListEquals(Arrays.asList("java", "-cp", "FRAMEWORK_CP:" + stormjar.getAbsolutePath(), "-Dlogging.sensitivity=S3", "-Dlogfile.name=worker.log", "-Dstorm.home=" + stormHome, "-Dworkers.artifacts=" + stormLocal, "-Dstorm.id=" + topoId, "-Dworker.id=" + workerId, "-Dworker.port=" + port, "-Dstorm.log.dir=" + stormLogDir, "-DLog4jContextSelector=org.apache.logging.log4j.core.selector.BasicContextSelector", "-Dstorm.local.dir=" + stormLocal, "-Dworker.memory_limit_mb=768", "-Dlog4j.configurationFile=" + workerConf, "org.apache.storm.LogWriter", "java", "-server", "-Dlogging.sensitivity=S3", "-Dlogfile.name=worker.log", "-Dstorm.home=" + stormHome, "-Dworkers.artifacts=" + stormLocal, "-Dstorm.id=" + topoId, "-Dworker.id=" + workerId, "-Dworker.port=" + port, "-Dstorm.log.dir=" + stormLogDir, "-DLog4jContextSelector=org.apache.logging.log4j.core.selector.BasicContextSelector", "-Dstorm.local.dir=" + stormLocal, "-Dworker.memory_limit_mb=768", "-Dlog4j.configurationFile=" + workerConf, "-Dtesting=true", "-Djava.library.path=JLP", "-Dstorm.conf.file=", "-Dstorm.options=", "-Djava.io.tmpdir=" + workerTmpDir, "-cp", "FRAMEWORK_CP:" + stormjar.getAbsolutePath(), "org.apache.storm.daemon.worker", topoId, "SUPERVISOR", String.valueOf(port), workerId), cmd.cmd);
        assertEquals(new File(workerRoot), cmd.pwd);
    }, ConfigUtils.STORM_HOME, stormHome, "storm.log.dir", stormLogDir);
}

Example 73 with StormTopology

use of org.apache.storm.generated.StormTopology in project metron by apache.

the class FluxTopologyComponent method startTopology.

private void startTopology(String topologyName, File topologyLoc, File templateFile, Properties properties) throws IOException, ClassNotFoundException, NoSuchMethodException, InvocationTargetException, InstantiationException, IllegalAccessException, TException, NoSuchFieldException {
    TopologyDef topologyDef = loadYaml(topologyName, topologyLoc, templateFile, properties);
    Config conf = FluxBuilder.buildConfig(topologyDef);
    ExecutionContext context = new ExecutionContext(topologyDef, conf);
    StormTopology topology = FluxBuilder.buildTopology(context);
    assertNotNull(topology);
    topology.validate();
    try {
        stormCluster.submitTopology(topologyName, conf, topology);
    } catch (Exception nne) {
        try {
            Thread.sleep(2000);
        } catch (InterruptedException e) {
        }
        stormCluster.submitTopology(topologyName, conf, topology);
    }
}

Example 74 with StormTopology

use of org.apache.storm.generated.StormTopology in project open-kilda by telstra.

the class ControllerToSpeakerProxyBoltTest method setUp.

@Before
public void setUp() {
    Set<String> regions = new HashSet<>();
    regions.add(REGION_ONE);
    regions.add(REGION_TWO);
    subject = new ControllerToSpeakerProxyBolt(TARGET_TOPIC, regions, Duration.ofSeconds(900));
    when(topologyContext.getThisTaskId()).thenReturn(1);
    subject.prepare(topologyConfig, topologyContext, outputCollector);
    StormTopology topology = mock(StormTopology.class);
    Map<Integer, String> taskToComponent = ImmutableMap.of(TASK_ID_SPOUT, ComponentType.SPEAKER_KAFKA_SPOUT, SWITCH_MONITOR_BOLT, SwitchMonitorBolt.BOLT_ID, ZOOKEEPER_SPOUT, ZooKeeperSpout.SPOUT_ID);
    Map<String, Map<String, Fields>> componentToFields = ImmutableMap.of(ComponentType.SPEAKER_KAFKA_SPOUT, ImmutableMap.of(Utils.DEFAULT_STREAM_ID, new Fields(KafkaRecordTranslator.FIELD_ID_KEY, KafkaRecordTranslator.FIELD_ID_PAYLOAD, AbstractBolt.FIELD_ID_CONTEXT)), SwitchMonitorBolt.BOLT_ID, ImmutableMap.of(SwitchMonitorBolt.STREAM_REGION_MAPPING_ID, SwitchMonitorBolt.STREAM_REGION_MAPPING_FIELDS), ZooKeeperSpout.SPOUT_ID, ImmutableMap.of(Utils.DEFAULT_STREAM_ID, new Fields(ZooKeeperSpout.FIELD_ID_LIFECYCLE_EVENT, ZooKeeperSpout.FIELD_ID_CONTEXT)));
    generalTopologyContext = new GeneralTopologyContext(topology, topologyConfig, taskToComponent, Collections.emptyMap(), componentToFields, "dummy");
}

Example 75 with StormTopology

use of org.apache.storm.generated.StormTopology in project open-kilda by telstra.

the class OpenTsdbTopologyTest method shouldSuccessfulSendDatapoint.

@Test
public void shouldSuccessfulSendDatapoint() {
    Datapoint datapoint = new Datapoint("metric", timestamp, Collections.emptyMap(), 123);
    MockedSources sources = new MockedSources();
    Testing.withTrackedCluster(clusterParam, (cluster) -> {
        OpenTsdbTopology topology = new OpenTsdbTopology(makeLaunchEnvironment(getProperties()));
        sources.addMockData(ZooKeeperSpout.SPOUT_ID, new Values(LifecycleEvent.builder().signal(Signal.NONE).build(), null));
        sources.addMockData(OpenTsdbTopology.OTSDB_SPOUT_ID, new Values(null, datapoint));
        completeTopologyParam.setMockedSources(sources);
        StormTopology stormTopology = topology.createTopology();
        stormTopology.get_bolts().remove(ZooKeeperBolt.BOLT_ID);
        activateDatapointParserBolt(stormTopology);
        Map result = Testing.completeTopology(cluster, stormTopology, completeTopologyParam);
    });
    // verify that request is sent to OpenTSDB server
    mockServer.verify(REQUEST, VerificationTimes.exactly(1));
}