Search in sources :

Example 1 with Setting

use of com.dtstack.taier.develop.common.template.Setting in project Taier by DTStack.

the class DatasourceService method getSyncSql.

/**
 * 配置或修改离线任务
 *
 * @param isFilter 获取数据同步脚本时候是否进行过滤用户名密码操作
 * @return
 * @throws IOException
 */
public String getSyncSql(final TaskResourceParam param, boolean isFilter) {
    // 来源集合
    final Map<String, Object> sourceMap = param.getSourceMap();
    // 目标集合
    final Map<String, Object> targetMap = param.getTargetMap();
    // 流控、错误集合
    final Map<String, Object> settingMap = param.getSettingMap();
    try {
        this.setReaderJson(sourceMap, param.getId(), param.getTenantId(), isFilter);
        this.setWriterJson(targetMap, param.getId(), param.getTenantId(), isFilter);
        Reader reader = null;
        Writer writer = null;
        Setting setting = null;
        final Integer sourceType = Integer.parseInt(sourceMap.get("dataSourceType").toString());
        final Integer targetType = Integer.parseInt(targetMap.get("dataSourceType").toString());
        if (!this.checkDataSourcePermission(sourceType, EDataSourcePermission.READ.getType())) {
            throw new RdosDefineException(ErrorCode.SOURCE_CAN_NOT_AS_INPUT);
        }
        if (!this.checkDataSourcePermission(targetType, EDataSourcePermission.WRITE.getType())) {
            throw new RdosDefineException(ErrorCode.SOURCE_CAN_NOT_AS_OUTPUT);
        }
        final List<Long> sourceIds = (List<Long>) sourceMap.get("sourceIds");
        final List<Long> targetIds = (List<Long>) targetMap.get("sourceIds");
        reader = this.syncReaderBuild(sourceType, sourceMap, sourceIds);
        writer = this.syncWriterBuild(targetType, targetIds, targetMap, reader);
        setting = PublicUtil.objectToObject(settingMap, DefaultSetting.class);
        // 检查有效性
        if (writer instanceof HiveWriter) {
            final HiveWriter hiveWriter = (HiveWriter) writer;
            if (!hiveWriter.isValid()) {
                throw new RdosDefineException(hiveWriter.getErrMsg());
            }
        }
        if (param.getCreateModel() == TaskCreateModelType.TEMPLATE.getType()) {
            // 脚本模式直接返回
            return this.getJobText(this.putDefaultEmptyValueForReader(sourceType, reader), this.putDefaultEmptyValueForWriter(targetType, writer), this.putDefaultEmptyValueForSetting(setting));
        }
        // 获得数据同步job.xml的配置
        final String jobXml = this.getJobText(reader, writer, setting);
        final String parserXml = this.getParserText(sourceMap, targetMap, settingMap);
        final JSONObject sql = new JSONObject(3);
        sql.put("job", jobXml);
        sql.put("parser", parserXml);
        sql.put("createModel", TaskCreateModelType.GUIDE.getType());
        this.batchTaskParamService.checkParams(this.batchTaskParamService.checkSyncJobParams(sql.toJSONString()), param.getTaskVariables());
        return sql.toJSONString();
    } catch (final Exception e) {
        LOGGER.error("", e);
        throw new RdosDefineException("解析同步任务失败: " + e.getMessage(), ErrorCode.SERVER_EXCEPTION);
    }
}
Also used : HiveWriter(com.dtstack.taier.develop.utils.develop.sync.template.HiveWriter) RdosDefineException(com.dtstack.taier.common.exception.RdosDefineException) DefaultSetting(com.dtstack.taier.develop.utils.develop.sync.template.DefaultSetting) Setting(com.dtstack.taier.develop.common.template.Setting) HBaseReader(com.dtstack.taier.develop.utils.develop.sync.template.HBaseReader) AwsS3Reader(com.dtstack.taier.develop.utils.develop.sync.template.AwsS3Reader) MongoDbReader(com.dtstack.taier.develop.utils.develop.sync.template.MongoDbReader) OdpsReader(com.dtstack.taier.develop.utils.develop.sync.template.OdpsReader) FtpReader(com.dtstack.taier.develop.utils.develop.sync.template.FtpReader) EsReader(com.dtstack.taier.develop.utils.develop.sync.template.EsReader) HiveReader(com.dtstack.taier.develop.utils.develop.sync.template.HiveReader) HDFSReader(com.dtstack.taier.develop.utils.develop.sync.template.HDFSReader) CarbonDataReader(com.dtstack.taier.develop.utils.develop.sync.template.CarbonDataReader) Reader(com.dtstack.taier.develop.common.template.Reader) InfluxDBReader(com.dtstack.taier.develop.utils.develop.sync.template.InfluxDBReader) RDBReader(com.dtstack.taier.develop.utils.develop.sync.template.RDBReader) SftpException(com.jcraft.jsch.SftpException) RdosDefineException(com.dtstack.taier.common.exception.RdosDefineException) IOException(java.io.IOException) PubSvcDefineException(com.dtstack.taier.common.exception.PubSvcDefineException) DtCenterDefException(com.dtstack.taier.common.exception.DtCenterDefException) JSONObject(com.alibaba.fastjson.JSONObject) DefaultSetting(com.dtstack.taier.develop.utils.develop.sync.template.DefaultSetting) JSONObject(com.alibaba.fastjson.JSONObject) ArrayList(java.util.ArrayList) List(java.util.List) HDFSWriter(com.dtstack.taier.develop.utils.develop.sync.template.HDFSWriter) CarbonDataWriter(com.dtstack.taier.develop.utils.develop.sync.template.CarbonDataWriter) EsWriter(com.dtstack.taier.develop.utils.develop.sync.template.EsWriter) OdpsWriter(com.dtstack.taier.develop.utils.develop.sync.template.OdpsWriter) HiveWriter(com.dtstack.taier.develop.utils.develop.sync.template.HiveWriter) RedisWriter(com.dtstack.taier.develop.utils.develop.sync.template.RedisWriter) AwsS3Writer(com.dtstack.taier.develop.utils.develop.sync.template.AwsS3Writer) HBaseWriter(com.dtstack.taier.develop.utils.develop.sync.template.HBaseWriter) RDBWriter(com.dtstack.taier.develop.utils.develop.sync.template.RDBWriter) InceptorWriter(com.dtstack.taier.develop.utils.develop.sync.template.InceptorWriter) MongoDbWriter(com.dtstack.taier.develop.utils.develop.sync.template.MongoDbWriter) FtpWriter(com.dtstack.taier.develop.utils.develop.sync.template.FtpWriter) Writer(com.dtstack.taier.develop.common.template.Writer)

Example 2 with Setting

use of com.dtstack.taier.develop.common.template.Setting in project Taier by DTStack.

the class JobTemplate method toJobJsonString.

public String toJobJsonString() {
    Reader reader = newReader();
    Writer writer = newWrite();
    Setting setting = newSetting();
    JSONObject content = new JSONObject(2);
    content.put("reader", reader.toReaderJson());
    content.put("writer", writer.toWriterJson());
    JSONObject jobJson = new JSONObject(2);
    jobJson.put("content", Lists.newArrayList(content));
    jobJson.put("setting", setting.toSettingJson());
    StringBuilder job = new StringBuilder();
    job.append("{ \"job\":");
    job.append(jobJson.toJSONString());
    job.append(" }");
    return job.toString();
}
Also used : JSONObject(com.alibaba.fastjson.JSONObject) Setting(com.dtstack.taier.develop.common.template.Setting) Reader(com.dtstack.taier.develop.common.template.Reader) Writer(com.dtstack.taier.develop.common.template.Writer)

Aggregations

JSONObject (com.alibaba.fastjson.JSONObject)2 Reader (com.dtstack.taier.develop.common.template.Reader)2 Setting (com.dtstack.taier.develop.common.template.Setting)2 Writer (com.dtstack.taier.develop.common.template.Writer)2 DtCenterDefException (com.dtstack.taier.common.exception.DtCenterDefException)1 PubSvcDefineException (com.dtstack.taier.common.exception.PubSvcDefineException)1 RdosDefineException (com.dtstack.taier.common.exception.RdosDefineException)1 AwsS3Reader (com.dtstack.taier.develop.utils.develop.sync.template.AwsS3Reader)1 AwsS3Writer (com.dtstack.taier.develop.utils.develop.sync.template.AwsS3Writer)1 CarbonDataReader (com.dtstack.taier.develop.utils.develop.sync.template.CarbonDataReader)1 CarbonDataWriter (com.dtstack.taier.develop.utils.develop.sync.template.CarbonDataWriter)1 DefaultSetting (com.dtstack.taier.develop.utils.develop.sync.template.DefaultSetting)1 EsReader (com.dtstack.taier.develop.utils.develop.sync.template.EsReader)1 EsWriter (com.dtstack.taier.develop.utils.develop.sync.template.EsWriter)1 FtpReader (com.dtstack.taier.develop.utils.develop.sync.template.FtpReader)1 FtpWriter (com.dtstack.taier.develop.utils.develop.sync.template.FtpWriter)1 HBaseReader (com.dtstack.taier.develop.utils.develop.sync.template.HBaseReader)1 HBaseWriter (com.dtstack.taier.develop.utils.develop.sync.template.HBaseWriter)1 HDFSReader (com.dtstack.taier.develop.utils.develop.sync.template.HDFSReader)1 HDFSWriter (com.dtstack.taier.develop.utils.develop.sync.template.HDFSWriter)1