Examples with RETLStormCli com.ds.retl.cli.RETLStormCli used on opensource projects

Example 1 with RETLStormCli

use of com.ds.retl.cli.RETLStormCli in project main by JohnPeng739.

the class TopologyManageServiceImpl method submit.

/**
 * 提交一个拓扑实体对象到Storm集群中
 *
 * @param topology   拓扑实体对象
 * @param simulation 是否仿真
 * @return 成功返回拓扑实体对象
 * @throws UserInterfaceErrorException 提交过程中发生的异常
 */
private Topology submit(Topology topology, boolean simulation) throws UserInterfaceErrorException {
    String topologyName = topology.getName();
    if (!simulation) {
        // 检查拓扑名字是否已经被部署到Storm集群中
        JSONObject found = foundSubmitedTopology(topologyName);
        if (found != null) {
            try {
                topology.setSubmitted(false);
                topology.setSubmittedTime(new Date().getTime());
                topology.setSubmitInfo("同名的计算拓扑已经被部署到集群中，不能重复部署。");
                accessor.save(topology);
            } catch (EntityAccessException ex) {
                if (logger.isErrorEnabled()) {
                    logger.error("Save submit information fail.", ex);
                }
            }
            throw new UserInterfaceErrorException(UserInterfaceErrors.TOPOLOGY_ALREADY_SUBMITTED);
        }
    }
    String confStr = topology.getTopologyContent();
    if (StringUtils.isBlank(confStr)) {
        if (logger.isErrorEnabled()) {
            logger.error(String.format("Topology's content is blank, name: %s.", topologyName));
        }
        throw new UserInterfaceErrorException(UserInterfaceErrors.SYSTEM_ILLEGAL_PARAM);
    }
    JSONObject topologyJson = prepareTopologyConfigJsonObject(confStr);
    if (topology == null) {
        if (logger.isErrorEnabled()) {
            logger.error(String.format("Parse JSON object fail, conf: %s.", confStr));
        }
        throw new UserInterfaceErrorException(UserInterfaceErrors.SYSTEM_ILLEGAL_PARAM);
    }
    if (!simulation) {
        String stormHome = env.getProperty("storm.home"), stormBin = env.getProperty("storm.bin"), retlHome = env.getProperty("storm.retl"), retlPlatform = env.getProperty("storm.retl.platform"), retlDeps = env.getProperty("storm.retl.deps"), retlConf = env.getProperty("storm.retl.conf");
        String configName = String.format("%s/%s/%s.json", retlHome, retlConf, topologyName);
        File file = new File(configName);
        File parent = file.getParentFile();
        if (!parent.exists()) {
            parent.mkdirs();
        }
        try (BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(file)))) {
            bw.write(JSON.toJSONString(topologyJson, true));
        } catch (IOException ex) {
            if (logger.isErrorEnabled()) {
                logger.error(ex);
            }
            throw new UserInterfaceErrorException(UserInterfaceErrors.SYSTEM_FILE_OPERATE_FAIL);
        }
        try {
            int timeout = 60;
            String submitInfo = new RETLStormCli(stormHome, stormBin, retlHome, retlPlatform, retlDeps).deploy(configName, timeout);
            topology.setSubmitInfo(submitInfo);
            for (int index = 0; index < timeout; index++) {
                JSONObject submittedTopology = foundSubmitedTopology(topologyName);
                if (submittedTopology != null) {
                    topology.setSubmittedTime(new Date().getTime());
                    String topologyId = submittedTopology.getString("id");
                    topology.setTopologyId(topologyId);
                    topology.setSubmitted(true);
                    topology = accessor.save(topology);
                    operateLogService.writeLog(String.format("成功提交[%s]计算拓扑。", topologyName));
                    return topology;
                } else {
                    try {
                        Thread.sleep(1000);
                    } catch (InterruptedException ex) {
                        if (logger.isErrorEnabled()) {
                            logger.error(ex);
                        }
                    }
                }
            }
            if (logger.isDebugEnabled()) {
                logger.debug("The topology can be found in cluster after wait 10s.");
            }
            topology.setSubmittedTime(new Date().getTime());
            topology.setSubmitted(false);
            accessor.save(topology);
            operateLogService.writeLog(String.format("提交[%s]计算拓扑未成功。", topologyName));
            throw new UserInterfaceErrorException(UserInterfaceErrors.TOPOLOGY_SUBMIT_FAIL);
        } catch (EntityAccessException ex) {
            if (logger.isErrorEnabled()) {
                logger.error(ex);
            }
            throw new UserInterfaceErrorException(UserInterfaceErrors.DB_OPERATE_FAIL);
        }
    } else {
        // 本地仿真运行
        new ETLTopologyBuilder().buildTopology(false, topologyJson);
        return topology;
    }
}