Examples with RdosDefineException com.dtstack.taier.common.exception.RdosDefineException used on opensource projects

Example 26 with RdosDefineException

use of com.dtstack.taier.common.exception.RdosDefineException in project Taier by DTStack.

the class HiveReader method toReaderJson.

@Override
public JSONObject toReaderJson() {
    try {
        inferHdfsParams();
        HDFSReader hdfsReader = new HDFSReader();
        hdfsReader.setHadoopConfig(hadoopConfig);
        hdfsReader.setFileType(fileType);
        hdfsReader.setFieldDelimiter(fieldDelimiter);
        hdfsReader.setColumn(column);
        hdfsReader.setDefaultFS(defaultFS);
        hdfsReader.setEncoding(encoding);
        hdfsReader.setExtralConfig(super.getExtralConfig());
        hdfsReader.setSourceIds(getSourceIds());
        if (StringUtils.isNotEmpty(partition)) {
            hdfsReader.setPartition(partition);
            hdfsReader.setFileName(partition);
        }
        if (StringUtils.isNotEmpty(partition)) {
            hdfsReader.setPartition(partition);
        }
        if (MapUtils.isNotEmpty(sftpConf)) {
            hdfsReader.setSftpConf(sftpConf);
        }
        if (StringUtils.isNotEmpty(remoteDir)) {
            hdfsReader.setRemoteDir(remoteDir);
        }
        if (StringUtils.isNotEmpty(table)) {
            hdfsReader.setTable(table);
        }
        if (StringUtils.isNotEmpty(jdbcUrl)) {
            hdfsReader.setJdbcUrl(jdbcUrl);
        }
        if (StringUtils.isNotEmpty(username)) {
            hdfsReader.setUsername(username);
        }
        if (StringUtils.isNotEmpty(password)) {
            hdfsReader.setPassword(password);
        }
        hdfsReader.setPath(path == null ? "" : path.trim());
        return hdfsReader.toReaderJson();
    } catch (Exception ex) {
        throw new RdosDefineException(ex.getCause().getMessage());
    }
}

Example 27 with RdosDefineException

use of com.dtstack.taier.common.exception.RdosDefineException in project Taier by DTStack.

the class HiveWriter method checkFormat.

@Override
public void checkFormat(JSONObject data) {
    JSONObject parameter = data.getJSONObject("parameter");
    if (StringUtils.isEmpty(parameter.getString("path"))) {
        throw new RdosDefineException("path 不能为空");
    }
    JSONArray column = parameter.getJSONArray("column");
    if (column == null || column.isEmpty()) {
        throw new RdosDefineException("column 不能为空");
    }
    for (Object o : column) {
        if (o instanceof String) {
            throw new RdosDefineException("column 必须为对象数组 : [{\"name\":\"id\",\"type\":\"int\"}]");
        }
    }
}

Example 28 with RdosDefineException

use of com.dtstack.taier.common.exception.RdosDefineException in project Taier by DTStack.

the class RDBReader method toReaderJson.

@Override
public JSONObject toReaderJson() {
    JSONObject parameter = new JSONObject(true);
    parameter.put("username", this.getUsername());
    parameter.put("password", this.getPassword());
    parameter.put("splitPk", this.getSplitPK());
    parameter.put("where", this.getWhere());
    parameter.put("column", ColumnUtil.getColumns(this.getColumn(), PluginName.MySQLD_R));
    if (this.getJdbcUrl() != null) {
        JSONObject conn = new JSONObject(true);
        conn.put("jdbcUrl", Arrays.asList(this.getJdbcUrl()));
        conn.put("password", this.getPassword());
        conn.put("username", this.getUsername());
        conn.put("table", this.getTable());
        conn.put("sourceId", this.getSourceId());
        conn.put("schema", isNeedSchema());
        connections = new JSONArray();
        connections.add(conn);
    }
    if (connections != null && connections.size() > 0) {
        Map<String, Object> conn = (Map) connections.get(0);
        this.setJdbcUrl(((List<String>) conn.get("jdbcUrl")).get(0));
        String pass = Objects.isNull(conn.get("password")) ? "" : conn.get("password").toString();
        this.setPassword(pass);
        this.setUsername(MapUtils.getString(conn, "username"));
        if (conn.get("table") instanceof String) {
            this.setTable(Arrays.asList((String) conn.get("table")));
        } else {
            this.setTable((List<String>) conn.get("table"));
        }
    }
    boolean isMultiTable = (CollectionUtils.isNotEmpty(this.getConnections()) && this.getConnections().size() > 1) || (CollectionUtils.isNotEmpty(this.getTable()) && this.getTable().size() > 1);
    // 增量配置
    parameter.put("increColumn", Optional.ofNullable(this.getIncreColumn()).orElse(""));
    parameter.put("startLocation", "");
    parameter.put("connection", connections);
    JSONObject reader = new JSONObject(true);
    DataSourceType dataSourceType = DataSourceType.getSourceType(getType());
    switch(dataSourceType) {
        case MySQL:
        case MySQL8:
        case MySQLPXC:
        case TiDB:
            if (isMultiTable) {
                reader.put("name", PluginName.MySQLD_R);
            } else {
                reader.put("name", PluginName.MySQL_R);
            }
            break;
        case Clickhouse:
            reader.put("name", PluginName.Clickhouse_R);
            break;
        case Polardb_For_MySQL:
            reader.put("name", PluginName.Polardb_for_MySQL_R);
            break;
        case Oracle:
            reader.put("name", PluginName.Oracle_R);
            break;
        case SQLServer:
            reader.put("name", PluginName.SQLServer_R);
            break;
        case HIVE:
        case HIVE3X:
        case HIVE1X:
            reader.put("name", PluginName.Hive_R);
            break;
        case PostgreSQL:
        case LIBRA:
            reader.put("name", PluginName.PostgreSQL_R);
            break;
        case DB2:
            reader.put("name", PluginName.DB2_R);
            break;
        case GBase_8a:
            reader.put("name", PluginName.GBase_R);
            break;
        case Phoenix:
            reader.put("name", PluginName.Phoenix_R);
            break;
        case PHOENIX5:
            reader.put("name", PluginName.Phoenix5_R);
            break;
        case DMDB:
            reader.put("name", PluginName.DM_R);
            break;
        case GREENPLUM6:
            reader.put("name", PluginName.GREENPLUM_R);
            break;
        case KINGBASE8:
            reader.put("name", PluginName.KINGBASE_R);
            break;
        case ADB_FOR_PG:
            reader.put("name", PluginName.ADB_FOR_PG_R);
            break;
        default:
            throw new RdosDefineException(ErrorCode.CAN_NOT_FITABLE_SOURCE_TYPE);
    }
    parameter.put("customSql", Optional.ofNullable(getCustomSql()).orElse(""));
    parameter.put("sourceIds", getSourceIds());
    parameter.putAll(super.getExtralConfigMap());
    reader.put("parameter", parameter);
    return reader;
}

Example 29 with RdosDefineException

use of com.dtstack.taier.common.exception.RdosDefineException in project Taier by DTStack.

the class RDBReader method checkFormat.

@Override
public void checkFormat(JSONObject data) {
    String name = data.getString("name");
    data = data.getJSONObject("parameter");
    if (data.get("connection") == null) {
        throw new RdosDefineException("connection 不能为空");
    }
    if (!(data.get("connection") instanceof JSONArray)) {
        throw new RdosDefineException("connection 必须为数组格式");
    }
    JSONArray connections = data.getJSONArray("connection");
    if (connections.isEmpty()) {
        throw new RdosDefineException("connection 不能为空");
    }
    if (!name.equals(PluginName.MySQLD_R) && connections.size() > 1) {
        throw new RdosDefineException("暂不支持多个数据源写入");
    }
    for (Object connection : connections) {
        JSONObject conn = (JSONObject) connection;
        if (StringUtils.isEmpty(conn.getString("jdbcUrl"))) {
            throw new RdosDefineException("jdbcUrl 不能为空");
        }
        if (conn.get("table") == null) {
            throw new RdosDefineException("table 不能为空");
        }
        if (!(conn.get("table") instanceof JSONArray)) {
            throw new RdosDefineException("table 必须为数组格式");
        }
        JSONArray tables = conn.getJSONArray("table");
        if (tables.isEmpty()) {
            throw new RdosDefineException("table 不能为空");
        }
        if (name.equals(PluginName.Polardb_for_MySQL_R) && tables.size() > 1) {
            throw new RdosDefineException("polard 不支持多表");
        }
        if (!name.equals(PluginName.MySQLD_R) && tables.size() > 1) {
        // fixme 虽然只有mysql支持分库分表，但如果其他关系型数据库配置多个表，flinkx也是只取第一个表（隐藏逻辑）
        }
        for (Object table : tables) {
            if (!(table instanceof String)) {
                throw new RdosDefineException("table 必须为字符串数组格式");
            }
        }
    }
    if (data.get("column") == null) {
        throw new RdosDefineException("column 不能为空");
    }
    if (!(data.get("column") instanceof JSONArray)) {
        throw new RdosDefineException("column 必须为数组格式");
    }
    JSONArray column = data.getJSONArray("column");
    if (column.isEmpty()) {
        throw new RdosDefineException("column 不能为空");
    }
}

Example 30 with RdosDefineException

use of com.dtstack.taier.common.exception.RdosDefineException in project Taier by DTStack.

the class RDBWriter method toWriterJson.

@Override
public JSONObject toWriterJson() {
    JSONObject connection = new JSONObject(true);
    connection.put("jdbcUrl", this.getJdbcUrl());
    connection.put("table", this.getTable());
    connection.put("schema", isNeedSchema());
    JSONObject parameter = new JSONObject(true);
    parameter.put("username", this.getUsername());
    parameter.put("password", this.getPassword());
    parameter.put("connection", Lists.newArrayList(connection));
    parameter.put("session", StringUtils.isNotBlank(this.getSession()) ? Lists.newArrayList(this.getSession()) : Lists.newArrayList());
    parameter.put("preSql", StringUtils.isNotBlank(this.getPreSql()) ? Lists.newArrayList(this.getPreSql().trim().split(";")) : Lists.newArrayList());
    parameter.put("postSql", StringUtils.isNotBlank(this.getPostSql()) ? Lists.newArrayList(this.getPostSql().trim().split(";")) : Lists.newArrayList());
    parameter.put("writeMode", this.getWriteMode());
    parameter.put("column", ColumnUtil.getColumns(this.getColumn(), PluginName.MySQLD_R));
    parameter.put("sourceIds", getSourceIds());
    parameter.putAll(super.getExtralConfigMap());
    JSONObject writer = new JSONObject(true);
    DataSourceType dataSourceType = DataSourceType.getSourceType(getType());
    switch(dataSourceType) {
        case MySQL:
        case MySQL8:
        case MySQLPXC:
        case TiDB:
            writer.put("name", PluginName.MySQL_W);
            break;
        case Clickhouse:
            writer.put("name", PluginName.Clichhouse_W);
            break;
        case Polardb_For_MySQL:
            writer.put("name", PluginName.Polardb_for_MySQL_W);
            break;
        case Oracle:
            writer.put("name", PluginName.Oracle_W);
            break;
        case SQLServer:
            writer.put("name", PluginName.SQLServer_W);
            break;
        case HIVE:
        case HIVE1X:
            writer.put("name", PluginName.Hive_W);
            break;
        case PostgreSQL:
        case LIBRA:
            writer.put("name", PluginName.PostgreSQL_W);
            break;
        case DB2:
            writer.put("name", PluginName.DB2_W);
            break;
        case GBase_8a:
            writer.put("name", PluginName.GBase_W);
            break;
        case Phoenix:
            writer.put("name", PluginName.Phoenix_W);
            // 特殊处理写入模式，200302_3.10_beta2 只支持 upsert
            parameter.put("writeMode", StringUtils.isBlank(this.getWriteMode()) ? UPSERT : this.getWriteMode());
            break;
        case PHOENIX5:
            writer.put("name", PluginName.Phoenix5_W);
            parameter.put("writeMode", StringUtils.isBlank(this.getWriteMode()) ? UPSERT : this.getWriteMode());
            break;
        case DMDB:
            writer.put("name", PluginName.DM_W);
            break;
        case GREENPLUM6:
            writer.put("name", PluginName.GREENPLUM_W);
            break;
        case KINGBASE8:
            writer.put("name", PluginName.KINGBASE_W);
            break;
        case INCEPTOR:
            writer.put("name", PluginName.INCEPTOR_W);
            break;
        case ADB_FOR_PG:
            writer.put("name", PluginName.ADB_FOR_PG_W);
            break;
        default:
            throw new RdosDefineException(ErrorCode.CAN_NOT_FITABLE_SOURCE_TYPE);
    }
    writer.put("parameter", parameter);
    return writer;
}