Examples with StormTopology backtype.storm.generated.StormTopology used on opensource projects

Example 11 with StormTopology

use of backtype.storm.generated.StormTopology in project storm-lib by xumingming.

the class TestingApiDemo method testBasicTopology.

public void testBasicTopology() {
    MkClusterParam mkClusterParam = new MkClusterParam();
    mkClusterParam.setSupervisors(4);
    Config daemonConf = new Config();
    daemonConf.put(Config.STORM_LOCAL_MODE_ZMQ, false);
    mkClusterParam.setDaemonConf(daemonConf);
    /**
		 * This is a combination of <code>Testing.withLocalCluster</code> and <code>Testing.withSimulatedTime</code>.
		 */
    Testing.withSimulatedTimeLocalCluster(mkClusterParam, new TestJob() {

        @Override
        public void run(ILocalCluster cluster) {
            // build the test topology
            TopologyBuilder builder = new TopologyBuilder();
            builder.setSpout("1", new TestWordSpout(true), 3);
            builder.setBolt("2", new TestWordCounter(), 4).fieldsGrouping("1", new Fields("word"));
            builder.setBolt("3", new TestGlobalCount()).globalGrouping("1");
            builder.setBolt("4", new TestAggregatesCounter()).globalGrouping("2");
            StormTopology topology = builder.createTopology();
            // complete the topology
            // prepare the mock data
            MockedSources mockedSources = new MockedSources();
            mockedSources.addMockData("1", new Values("nathan"), new Values("bob"), new Values("joey"), new Values("nathan"));
            // prepare the config
            Config conf = new Config();
            conf.setNumWorkers(2);
            CompleteTopologyParam completeTopologyParam = new CompleteTopologyParam();
            completeTopologyParam.setMockedSources(mockedSources);
            completeTopologyParam.setStormConf(conf);
            /**
				 * TODO
				 */
            Map result = Testing.completeTopology(cluster, topology, completeTopologyParam);
            // check whether the result is right
            assertTrue(Testing.multiseteq(new Values(new Values("nathan"), new Values("bob"), new Values("joey"), new Values("nathan")), Testing.readTuples(result, "1")));
            assertTrue(Testing.multiseteq(new Values(new Values("nathan", 1), new Values("nathan", 2), new Values("bob", 1), new Values("joey", 1)), Testing.readTuples(result, "2")));
            assertTrue(Testing.multiseteq(new Values(new Values(1), new Values(2), new Values(3), new Values(4)), Testing.readTuples(result, "3")));
            assertTrue(Testing.multiseteq(new Values(new Values(1), new Values(2), new Values(3), new Values(4)), Testing.readTuples(result, "4")));
        }
    });
}

Example 12 with StormTopology

use of backtype.storm.generated.StormTopology in project jstorm by alibaba.

the class DoRebalanceTransitionCallback method execute.

@Override
public <T> Object execute(T... args) {
    boolean isSetTaskInfo = false;
    try {
        Boolean reassign = (Boolean) args[1];
        // args[0]:
        Map<Object, Object> conf = (Map<Object, Object>) args[2];
        // conf
        if (conf != null) {
            boolean isConfUpdate = false;
            Map stormConf = data.getConf();
            // Update topology code
            Map topoConf = StormConfig.read_nimbus_topology_conf(topologyid, data.getBlobStore());
            StormTopology rawOldTopology = StormConfig.read_nimbus_topology_code(topologyid, data.getBlobStore());
            StormTopology rawNewTopology = NimbusUtils.normalizeTopology(conf, rawOldTopology, true);
            StormTopology sysOldTopology = rawOldTopology.deepCopy();
            StormTopology sysNewTopology = rawNewTopology.deepCopy();
            if (conf.get(Config.TOPOLOGY_ACKER_EXECUTORS) != null) {
                Common.add_acker(topoConf, sysOldTopology);
                Common.add_acker(conf, sysNewTopology);
                int ackerNum = JStormUtils.parseInt(conf.get(Config.TOPOLOGY_ACKER_EXECUTORS));
                int oldAckerNum = JStormUtils.parseInt(topoConf.get(Config.TOPOLOGY_ACKER_EXECUTORS));
                LOG.info("Update acker from oldAckerNum=" + oldAckerNum + " to ackerNum=" + ackerNum);
                topoConf.put(Config.TOPOLOGY_ACKER_EXECUTORS, ackerNum);
                isConfUpdate = true;
            }
            // If scale-out, setup task info for new added tasks
            setTaskInfo(sysOldTopology, sysNewTopology);
            isSetTaskInfo = true;
            // If everything is OK, write topology code into disk
            StormConfig.write_nimbus_topology_code(topologyid, Utils.serialize(rawNewTopology), data);
            // Update topology conf if worker num has been updated
            Set<Object> keys = conf.keySet();
            Integer workerNum = JStormUtils.parseInt(conf.get(Config.TOPOLOGY_WORKERS));
            if (workerNum != null) {
                Integer oldWorkerNum = JStormUtils.parseInt(topoConf.get(Config.TOPOLOGY_WORKERS));
                topoConf.put(Config.TOPOLOGY_WORKERS, workerNum);
                isConfUpdate = true;
                LOG.info("Update worker num from " + oldWorkerNum + " to " + workerNum);
            }
            if (keys.contains(Config.ISOLATION_SCHEDULER_MACHINES)) {
                topoConf.put(Config.ISOLATION_SCHEDULER_MACHINES, conf.get(Config.ISOLATION_SCHEDULER_MACHINES));
            }
            if (isConfUpdate) {
                StormConfig.write_nimbus_topology_conf(topologyid, topoConf, data);
            }
        }
        TopologyAssignEvent event = new TopologyAssignEvent();
        event.setTopologyId(topologyid);
        event.setScratch(true);
        event.setOldStatus(oldStatus);
        event.setReassign(reassign);
        if (conf != null)
            event.setScaleTopology(true);
        TopologyAssign.push(event);
        event.waitFinish();
    } catch (Exception e) {
        LOG.error("do-rebalance error!", e);
        // Rollback the changes on ZK
        if (isSetTaskInfo) {
            try {
                StormClusterState clusterState = data.getStormClusterState();
                clusterState.remove_task(topologyid, newTasks);
            } catch (Exception e1) {
                LOG.error("Failed to rollback the changes on ZK for task-" + newTasks, e);
            }
        }
    }
    DelayStatusTransitionCallback delayCallback = new DelayStatusTransitionCallback(data, topologyid, oldStatus, StatusType.rebalancing, StatusType.done_rebalance);
    return delayCallback.execute();
}

Example 13 with StormTopology

use of backtype.storm.generated.StormTopology in project jstorm by alibaba.

the class ServiceHandler method restart.

@Override
public void restart(String name, String jsonConf) throws TException, NotAliveException, InvalidTopologyException, TopologyAssignException {
    LOG.info("Begin to restart " + name + ", new configuration:" + jsonConf);
    // 1. get topologyId
    StormClusterState stormClusterState = data.getStormClusterState();
    String topologyId;
    try {
        topologyId = Cluster.get_topology_id(stormClusterState, name);
    } catch (Exception e2) {
        topologyId = null;
    }
    if (topologyId == null) {
        LOG.info("No topology of " + name);
        throw new NotAliveException("No topology of " + name);
    }
    // Restart the topology: Deactivate -> Kill -> Submit
    // 2. Deactivate
    deactivate(name);
    JStormUtils.sleepMs(5000);
    LOG.info("Deactivate " + name);
    // 3. backup old jar/configuration/topology
    StormTopology topology;
    Map topologyConf;
    String topologyCodeLocation = null;
    try {
        topology = StormConfig.read_nimbus_topology_code(topologyId, data.getBlobStore());
        topologyConf = StormConfig.read_nimbus_topology_conf(topologyId, data.getBlobStore());
        if (jsonConf != null) {
            Map<Object, Object> newConf = (Map<Object, Object>) JStormUtils.from_json(jsonConf);
            topologyConf.putAll(newConf);
        }
        // copy storm files back to inbox from blob store
        String parent = StormConfig.masterInbox(conf);
        topologyCodeLocation = parent + PathUtils.SEPERATOR + topologyId;
        FileUtils.forceMkdir(new File(topologyCodeLocation));
        FileUtils.cleanDirectory(new File(topologyCodeLocation));
        copyBackToInbox(topologyId, topologyCodeLocation);
        LOG.info("Successfully read old jar/conf/topology " + name);
        notifyTopologyActionListener(name, "restart");
    } catch (Exception e) {
        LOG.error("Failed to read old jar/conf/topology", e);
        if (topologyCodeLocation != null) {
            try {
                PathUtils.rmr(topologyCodeLocation);
            } catch (IOException ignored) {
            }
        }
        throw new TException("Failed to read old jar/conf/topology ");
    }
    // 4. Kill
    // directly use remove command to kill, more stable than issue kill cmd
    RemoveTransitionCallback killCb = new RemoveTransitionCallback(data, topologyId);
    killCb.execute(new Object[0]);
    LOG.info("Successfully kill the topology " + name);
    // send metric events
    KillTopologyEvent.pushEvent(topologyId);
    // 5. submit
    try {
        submitTopology(name, topologyCodeLocation, JStormUtils.to_json(topologyConf), topology);
    } catch (AlreadyAliveException e) {
        LOG.info("Failed to kill the topology" + name);
        throw new TException("Failed to kill the topology" + name);
    } finally {
        try {
            PathUtils.rmr(topologyCodeLocation);
        } catch (IOException ignored) {
        }
    }
}

Example 14 with StormTopology

use of backtype.storm.generated.StormTopology in project jstorm by alibaba.

the class ServiceHandler method getTopology.

/**
     * get StormTopology throw deserialize local files
     *
     * @param id String: topology id
     * @return StormTopology
     */
@Override
public StormTopology getTopology(String id) throws NotAliveException, TException {
    StormTopology topology;
    try {
        StormTopology stormtopology = StormConfig.read_nimbus_topology_code(id, data.getBlobStore());
        if (stormtopology == null) {
            throw new NotAliveException("No topology of " + id);
        }
        Map<Object, Object> topologyConf = (Map<Object, Object>) StormConfig.read_nimbus_topology_conf(id, data.getBlobStore());
        topology = Common.system_topology(topologyConf, stormtopology);
    } catch (Exception e) {
        LOG.error("Failed to get topology " + id + ",", e);
        throw new TException("Failed to get system_topology");
    }
    return topology;
}

Example 15 with StormTopology

use of backtype.storm.generated.StormTopology in project jstorm by alibaba.

the class TopologyAssign method prepareTopologyAssign.

protected TopologyAssignContext prepareTopologyAssign(TopologyAssignEvent event) throws Exception {
    TopologyAssignContext ret = new TopologyAssignContext();
    String topologyId = event.getTopologyId();
    ret.setTopologyId(topologyId);
    int topoMasterId = nimbusData.getTasksHeartbeat().get(topologyId).get_topologyMasterId();
    ret.setTopologyMasterTaskId(topoMasterId);
    LOG.info("prepareTopologyAssign, topoMasterId={}", topoMasterId);
    Map<Object, Object> nimbusConf = nimbusData.getConf();
    Map<Object, Object> topologyConf = StormConfig.read_nimbus_topology_conf(topologyId, nimbusData.getBlobStore());
    StormTopology rawTopology = StormConfig.read_nimbus_topology_code(topologyId, nimbusData.getBlobStore());
    ret.setRawTopology(rawTopology);
    Map stormConf = new HashMap();
    stormConf.putAll(nimbusConf);
    stormConf.putAll(topologyConf);
    ret.setStormConf(stormConf);
    StormClusterState stormClusterState = nimbusData.getStormClusterState();
    // get all running supervisor, don't need callback to watch supervisor
    Map<String, SupervisorInfo> supInfos = Cluster.get_all_SupervisorInfo(stormClusterState, null);
    // init all AvailableWorkerPorts
    for (Entry<String, SupervisorInfo> supInfo : supInfos.entrySet()) {
        SupervisorInfo supervisor = supInfo.getValue();
        if (supervisor != null)
            supervisor.setAvailableWorkerPorts(supervisor.getWorkerPorts());
    }
    getAliveSupervsByHb(supInfos, nimbusConf);
    if (supInfos.size() == 0) {
        throw new FailedAssignTopologyException("Failed to make assignment " + topologyId + ", due to no alive supervisor");
    }
    Map<Integer, String> taskToComponent = Cluster.get_all_task_component(stormClusterState, topologyId, null);
    ret.setTaskToComponent(taskToComponent);
    // get taskids /ZK/tasks/topologyId
    Set<Integer> allTaskIds = taskToComponent.keySet();
    if (allTaskIds == null || allTaskIds.size() == 0) {
        String errMsg = "Failed to get all task ID list from /ZK-dir/tasks/" + topologyId;
        LOG.warn(errMsg);
        throw new IOException(errMsg);
    }
    ret.setAllTaskIds(allTaskIds);
    Set<Integer> aliveTasks = new HashSet<Integer>();
    // unstoppedTasks are tasks which are alive on no supervisor's(dead)
    // machine
    Set<Integer> unstoppedTasks = new HashSet<Integer>();
    Set<Integer> deadTasks = new HashSet<Integer>();
    Set<ResourceWorkerSlot> unstoppedWorkers = new HashSet<ResourceWorkerSlot>();
    Assignment existingAssignment = stormClusterState.assignment_info(topologyId, null);
    if (existingAssignment != null) {
        aliveTasks = getAliveTasks(topologyId, allTaskIds);
        /*
             * Check if the topology master task is alive first since all task 
             * heartbeat info is reported by topology master. 
             * If master is dead, do reassignment for topology master first.
             */
        if (aliveTasks.contains(topoMasterId) == false) {
            ResourceWorkerSlot worker = existingAssignment.getWorkerByTaskId(topoMasterId);
            deadTasks.addAll(worker.getTasks());
            Set<Integer> tempSet = new HashSet<Integer>(allTaskIds);
            tempSet.removeAll(deadTasks);
            aliveTasks.addAll(tempSet);
            aliveTasks.removeAll(deadTasks);
        } else {
            deadTasks.addAll(allTaskIds);
            deadTasks.removeAll(aliveTasks);
        }
        unstoppedTasks = getUnstoppedSlots(aliveTasks, supInfos, existingAssignment);
    }
    ret.setDeadTaskIds(deadTasks);
    ret.setUnstoppedTaskIds(unstoppedTasks);
    // Step 2: get all slots resource, free slots/ alive slots/ unstopped
    // slots
    getFreeSlots(supInfos, stormClusterState);
    ret.setCluster(supInfos);
    if (existingAssignment == null) {
        ret.setAssignType(TopologyAssignContext.ASSIGN_TYPE_NEW);
        try {
            AssignmentBak lastAssignment = stormClusterState.assignment_bak(event.getTopologyName());
            if (lastAssignment != null) {
                ret.setOldAssignment(lastAssignment.getAssignment());
            }
        } catch (Exception e) {
            LOG.warn("Fail to get old assignment", e);
        }
    } else {
        ret.setOldAssignment(existingAssignment);
        if (event.isScratch()) {
            ret.setAssignType(TopologyAssignContext.ASSIGN_TYPE_REBALANCE);
            ret.setIsReassign(event.isReassign());
            unstoppedWorkers = getUnstoppedWorkers(unstoppedTasks, existingAssignment);
            ret.setUnstoppedWorkers(unstoppedWorkers);
        } else {
            ret.setAssignType(TopologyAssignContext.ASSIGN_TYPE_MONITOR);
            unstoppedWorkers = getUnstoppedWorkers(aliveTasks, existingAssignment);
            ret.setUnstoppedWorkers(unstoppedWorkers);
        }
    }
    return ret;
}