Examples with ETLTopologyBuilder com.ds.retl.ETLTopologyBuilder used on opensource projects

Example 1 with ETLTopologyBuilder

use of com.ds.retl.ETLTopologyBuilder in project main by JohnPeng739.

the class TopologyManageServiceImpl method submit.

/**
 * 提交一个拓扑实体对象到Storm集群中
 *
 * @param topology   拓扑实体对象
 * @param simulation 是否仿真
 * @return 成功返回拓扑实体对象
 * @throws UserInterfaceErrorException 提交过程中发生的异常
 */
private Topology submit(Topology topology, boolean simulation) throws UserInterfaceErrorException {
    String topologyName = topology.getName();
    if (!simulation) {
        // 检查拓扑名字是否已经被部署到Storm集群中
        JSONObject found = foundSubmitedTopology(topologyName);
        if (found != null) {
            try {
                topology.setSubmitted(false);
                topology.setSubmittedTime(new Date().getTime());
                topology.setSubmitInfo("同名的计算拓扑已经被部署到集群中，不能重复部署。");
                accessor.save(topology);
            } catch (EntityAccessException ex) {
                if (logger.isErrorEnabled()) {
                    logger.error("Save submit information fail.", ex);
                }
            }
            throw new UserInterfaceErrorException(UserInterfaceErrors.TOPOLOGY_ALREADY_SUBMITTED);
        }
    }
    String confStr = topology.getTopologyContent();
    if (StringUtils.isBlank(confStr)) {
        if (logger.isErrorEnabled()) {
            logger.error(String.format("Topology's content is blank, name: %s.", topologyName));
        }
        throw new UserInterfaceErrorException(UserInterfaceErrors.SYSTEM_ILLEGAL_PARAM);
    }
    JSONObject topologyJson = prepareTopologyConfigJsonObject(confStr);
    if (topology == null) {
        if (logger.isErrorEnabled()) {
            logger.error(String.format("Parse JSON object fail, conf: %s.", confStr));
        }
        throw new UserInterfaceErrorException(UserInterfaceErrors.SYSTEM_ILLEGAL_PARAM);
    }
    if (!simulation) {
        String stormHome = env.getProperty("storm.home"), stormBin = env.getProperty("storm.bin"), retlHome = env.getProperty("storm.retl"), retlPlatform = env.getProperty("storm.retl.platform"), retlDeps = env.getProperty("storm.retl.deps"), retlConf = env.getProperty("storm.retl.conf");
        String configName = String.format("%s/%s/%s.json", retlHome, retlConf, topologyName);
        File file = new File(configName);
        File parent = file.getParentFile();
        if (!parent.exists()) {
            parent.mkdirs();
        }
        try (BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(file)))) {
            bw.write(JSON.toJSONString(topologyJson, true));
        } catch (IOException ex) {
            if (logger.isErrorEnabled()) {
                logger.error(ex);
            }
            throw new UserInterfaceErrorException(UserInterfaceErrors.SYSTEM_FILE_OPERATE_FAIL);
        }
        try {
            int timeout = 60;
            String submitInfo = new RETLStormCli(stormHome, stormBin, retlHome, retlPlatform, retlDeps).deploy(configName, timeout);
            topology.setSubmitInfo(submitInfo);
            for (int index = 0; index < timeout; index++) {
                JSONObject submittedTopology = foundSubmitedTopology(topologyName);
                if (submittedTopology != null) {
                    topology.setSubmittedTime(new Date().getTime());
                    String topologyId = submittedTopology.getString("id");
                    topology.setTopologyId(topologyId);
                    topology.setSubmitted(true);
                    topology = accessor.save(topology);
                    operateLogService.writeLog(String.format("成功提交[%s]计算拓扑。", topologyName));
                    return topology;
                } else {
                    try {
                        Thread.sleep(1000);
                    } catch (InterruptedException ex) {
                        if (logger.isErrorEnabled()) {
                            logger.error(ex);
                        }
                    }
                }
            }
            if (logger.isDebugEnabled()) {
                logger.debug("The topology can be found in cluster after wait 10s.");
            }
            topology.setSubmittedTime(new Date().getTime());
            topology.setSubmitted(false);
            accessor.save(topology);
            operateLogService.writeLog(String.format("提交[%s]计算拓扑未成功。", topologyName));
            throw new UserInterfaceErrorException(UserInterfaceErrors.TOPOLOGY_SUBMIT_FAIL);
        } catch (EntityAccessException ex) {
            if (logger.isErrorEnabled()) {
                logger.error(ex);
            }
            throw new UserInterfaceErrorException(UserInterfaceErrors.DB_OPERATE_FAIL);
        }
    } else {
        // 本地仿真运行
        new ETLTopologyBuilder().buildTopology(false, topologyJson);
        return topology;
    }
}

Example 2 with ETLTopologyBuilder

use of com.ds.retl.ETLTopologyBuilder in project main by JohnPeng739.

the class TestInsertData method testDataJJD1.

@Test
public void testDataJJD1() throws Exception {
    ETLTopologyBuilder builder = new ETLTopologyBuilder();
    JSONReader reader = new JSONReader(new InputStreamReader(this.getClass().getResourceAsStream("/sample-jdbc-2-jms.json")));
    JSONObject json = JSON.parseObject(reader.readString());
    reader.close();
    JSONArray config = json.getJSONArray("dataSources");
    JSONObject caches = json.getJSONObject("caches");
    JdbcManager.getManager().initManager(config, caches);
    Connection conn = JdbcManager.getManager().getConnection("dataSource1");
    PreparedStatement ps = conn.prepareStatement("select indexNo,id from cjd2 order by indexNo");
    ResultSet rs = ps.executeQuery();
    long index = 0, total = 0, errorNum = 0;
    while (rs.next()) {
        total++;
        long indexNo = rs.getLong(1);
        if (indexNo == index + 1) {
            index = indexNo;
            continue;
        }
        index = indexNo;
        errorNum++;
        System.out.println(String.format("indexNo: %d, id: %s.", indexNo, rs.getString(2)));
    }
    System.out.println(String.format("Total: %d, error: %d.", total, errorNum));
    rs.close();
    ps.close();
    conn.close();
}

Example 3 with ETLTopologyBuilder

use of com.ds.retl.ETLTopologyBuilder in project main by JohnPeng739.

the class TestInsertData method testBatch.

private void testBatch(String configFile, String table) throws Exception {
    ETLTopologyBuilder builder = new ETLTopologyBuilder();
    JSONReader reader = new JSONReader(new InputStreamReader(this.getClass().getResourceAsStream(configFile)));
    JSONObject json = JSON.parseObject(reader.readString());
    reader.close();
    JSONArray config = json.getJSONArray("dataSources");
    JSONObject caches = json.getJSONObject("caches");
    JdbcManager.getManager().initManager(config, caches);
    Connection conn = JdbcManager.getManager().getConnection("dataSource1");
    PreparedStatement ps = conn.prepareStatement(String.format("INSERT INTO %s(ID,CODE,NAME,FAMILIES,AGE,MONEY," + "BIRTHDAY,CREATEDTIME,UPDATEDTIME,EMAIL,MARRIED,INDEXNO) " + "VALUES(?,?,?,?,?,?,?,?,?,?,?,?)", table));
    for (int index = 1; index <= 1500; index++) {
        ps.setString(1, UUID.randomUUID().toString());
        ps.setString(2, "jjd");
        ps.setString(3, "Jjd Sample");
        ps.setInt(4, 5);
        ps.setInt(5, 43);
        ps.setDouble(6, 4567344.33);
        ps.setDate(7, new java.sql.Date(new Date().getTime()));
        // new java.sql.Timestamp(new Date().getTime()));
        ps.setTimestamp(8, null);
        // new java.sql.Timestamp(new Date().getTime()));
        ps.setTimestamp(9, null);
        ps.setString(10, "jjd@dscomm.com.cn");
        ps.setBoolean(11, true);
        ps.setLong(12, index);
        ps.executeUpdate();
        System.out.println(index);
    }
    ps.close();
}

Example 4 with ETLTopologyBuilder

use of com.ds.retl.ETLTopologyBuilder in project main by JohnPeng739.

the class TestInsertData method testSingle.

@Test
public void testSingle() throws Exception {
    ETLTopologyBuilder builder = new ETLTopologyBuilder();
    JSONReader reader = new JSONReader(new InputStreamReader(this.getClass().getResourceAsStream("/etl-topology-config-sample2.json")));
    JSONObject json = JSON.parseObject(reader.readString());
    reader.close();
    JSONArray config = json.getJSONArray("dataSources");
    JSONObject caches = json.getJSONObject("caches");
    JdbcManager.getManager().initManager(config, caches);
    Connection conn = JdbcManager.getManager().getConnection("dataSource1");
    PreparedStatement ps = conn.prepareStatement("INSERT INTO TB_SRC(ID, CODE, NAME, FAMILIES, AGE, MONEY, " + "BIRTHDAY, CREATED_TIME, UPDATED_TIME, EMAIL, MARRIED, INDEX_NO, SEX, DEPARTMENT_CODE, LAST_ORDER_ID) " + "VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)");
    ps.setString(1, UUID.randomUUID().toString());
    ps.setString(2, "jian");
    ps.setString(3, "Jian Peng");
    ps.setInt(4, 0);
    ps.setInt(5, 43);
    ps.setDouble(6, 4567344.33);
    ps.setDate(7, new java.sql.Date(new Date().getTime()));
    ps.setTimestamp(8, new java.sql.Timestamp(new Date().getTime()));
    ps.setTimestamp(9, new java.sql.Timestamp(new Date().getTime()));
    ps.setString(10, "pengmingxi@dscomm.com.cn");
    ps.setBoolean(11, true);
    ps.setLong(12, 1);
    ps.setString(13, "MALE");
    ps.setString(14, "kfsb");
    ps.setString(15, "ORDER_ID");
    ps.executeUpdate();
    Thread.sleep(10);
    ps.close();
}