Examples with JdbcSpout com.ds.retl.spout.JdbcSpout used on opensource projects

Example 1 with JdbcSpout

use of com.ds.retl.spout.JdbcSpout in project main by JohnPeng739.

the class ETLTopologyBuilder method createSpouts.

/**
 * 建立拓扑中的数据采集器
 *
 * @param spoutArray 采集器配置信息
 * @param builder    拓扑创建者对象
 */
private void createSpouts(JSONArray spoutArray, boolean isPersist, TopologyBuilder builder) {
    for (int index = 0; index < spoutArray.size(); index++) {
        JSONObject jsonSpout = spoutArray.getJSONObject(index);
        String name = jsonSpout.getString("name");
        String type = jsonSpout.getString("type");
        int parallelism = jsonSpout.getInteger("parallelism") != null ? jsonSpout.getInteger("parallelism") : 1;
        if (StringUtils.isBlank(name)) {
            String message = "Spout configuration error, not define the name of spout.";
            if (logger.isErrorEnabled()) {
                logger.error(message);
            }
            throw new IllegalArgumentException(message);
        }
        if (StringUtils.isBlank(type)) {
            String message = "Spout configuration error, not define the type of spout.";
            if (logger.isErrorEnabled()) {
                logger.error(message);
            }
            throw new IllegalArgumentException(message);
        }
        if ("jms".equalsIgnoreCase(type) || "jmsPULL".equalsIgnoreCase(type)) {
            // JMS Spout
            String method = jsonSpout.getString("method");
            JmsManager.Supported supported = JmsManager.Supported.valueOf(method);
            JSONObject configuration = jsonSpout.getJSONObject("configuration");
            try {
                BaseRichSpout spout = "JMS".equalsIgnoreCase(type) ? JmsManager.createJmsSpout(supported, configuration, isPersist) : JmsManager.createJmsPullSpout(supported, configuration, isPersist);
                builder.setSpout(name, spout, parallelism);
            } catch (JMSException ex) {
                if (logger.isErrorEnabled()) {
                    logger.error(String.format("Create JmsSpout fail, name: %s, method: %s, configuration: %s.", name, method, configuration.toJSONString()), ex);
                }
                throw new IllegalArgumentException(ex.getMessage(), ex);
            }
        } else if ("jdbc".equalsIgnoreCase(type)) {
            JSONObject configuration = jsonSpout.getJSONObject("configuration");
            JdbcSpout spout = new JdbcSpout(configuration);
            // JdbcSpout的并行度必须设置为1，防止数据库死锁
            builder.setSpout(name, spout, 1);
        } else {
            throw new IllegalArgumentException(String.format("Unsupported spout's type: %s.", type));
        }
    }
}