Examples with AdsException com.alibaba.datax.plugin.writer.adswriter.AdsException used on opensource projects

Example 1 with AdsException

use of com.alibaba.datax.plugin.writer.adswriter.AdsException in project DataX by alibaba.

the class AdsInsertUtil method getAdsTableColumnNames.

/*
     * 返回列顺序为ads建表列顺序
     * */
public static List<String> getAdsTableColumnNames(Configuration conf) {
    List<String> tableColumns = new ArrayList<String>();
    AdsHelper adsHelper = AdsUtil.createAdsHelper(conf);
    TableInfo tableInfo = null;
    String adsTable = conf.getString(Key.ADS_TABLE);
    try {
        tableInfo = adsHelper.getTableInfo(adsTable);
    } catch (AdsException e) {
        throw DataXException.asDataXException(AdsWriterErrorCode.GET_ADS_TABLE_MEATA_FAILED, e);
    }
    List<ColumnInfo> columnInfos = tableInfo.getColumns();
    for (ColumnInfo columnInfo : columnInfos) {
        tableColumns.add(columnInfo.getName());
    }
    LOG.info("table:[{}] all columns:[\n{}\n].", adsTable, StringUtils.join(tableColumns, ","));
    return tableColumns;
}

Example 2 with AdsException

use of com.alibaba.datax.plugin.writer.adswriter.AdsException in project DataX by alibaba.

the class AdsInsertUtil method getAdsTableInfo.

public static TableInfo getAdsTableInfo(Configuration conf) {
    AdsHelper adsHelper = AdsUtil.createAdsHelper(conf);
    TableInfo tableInfo = null;
    try {
        tableInfo = adsHelper.getTableInfo(conf.getString(Key.ADS_TABLE));
    } catch (AdsException e) {
        throw DataXException.asDataXException(AdsWriterErrorCode.GET_ADS_TABLE_MEATA_FAILED, e);
    }
    return tableInfo;
}

Example 3 with AdsException

use of com.alibaba.datax.plugin.writer.adswriter.AdsException in project DataX by alibaba.

the class AdsHelper method getTableInfo.

/**
     * Obtain the table meta information.
     * 
     * @param table The table
     * @return The table meta information
     * @throws com.alibaba.datax.plugin.writer.adswriter.AdsException
     */
public TableInfo getTableInfo(String table) throws AdsException {
    if (table == null) {
        throw new AdsException(AdsException.ADS_TABLEMETA_TABLE_NULL, "Table is null.", null);
    }
    if (adsURL == null) {
        throw new AdsException(AdsException.ADS_CONN_URL_NOT_SET, "ADS JDBC connection URL was not set.", null);
    }
    if (userName == null) {
        throw new AdsException(AdsException.ADS_CONN_USERNAME_NOT_SET, "ADS JDBC connection user name was not set.", null);
    }
    if (password == null) {
        throw new AdsException(AdsException.ADS_CONN_PASSWORD_NOT_SET, "ADS JDBC connection password was not set.", null);
    }
    if (schema == null) {
        throw new AdsException(AdsException.ADS_CONN_SCHEMA_NOT_SET, "ADS JDBC connection schema was not set.", null);
    }
    Connection connection = null;
    Statement statement = null;
    ResultSet rs = null;
    try {
        Class.forName("com.mysql.jdbc.Driver");
        String url = AdsUtil.prepareJdbcUrl(this.adsURL, this.schema, this.socketTimeout, this.suffix);
        Properties connectionProps = new Properties();
        connectionProps.put("user", userName);
        connectionProps.put("password", password);
        connection = DriverManager.getConnection(url, connectionProps);
        statement = connection.createStatement();
        // ads 表名、schema名不区分大小写, 提高用户易用性, 注意列顺序性
        String columnMetaSql = String.format("select ordinal_position,column_name,data_type,type_name,column_comment from information_schema.columns where lower(table_schema) = `'%s'` and lower(table_name) = `'%s'` order by ordinal_position", schema.toLowerCase(), table.toLowerCase());
        LOG.info(String.format("检查列信息sql语句:%s", columnMetaSql));
        rs = statement.executeQuery(columnMetaSql);
        TableInfo tableInfo = new TableInfo();
        List<ColumnInfo> columnInfoList = new ArrayList<ColumnInfo>();
        while (DBUtil.asyncResultSetNext(rs)) {
            ColumnInfo columnInfo = new ColumnInfo();
            columnInfo.setOrdinal(rs.getInt(1));
            columnInfo.setName(rs.getString(2));
            //columnInfo.setDataType(ColumnDataType.getDataType(rs.getInt(3))); //for ads version < 0.7
            //columnInfo.setDataType(ColumnDataType.getTypeByName(rs.getString(3).toUpperCase())); //for ads version 0.8
            //for ads version 0.8 & 0.7
            columnInfo.setDataType(ColumnDataType.getTypeByName(rs.getString(4).toUpperCase()));
            columnInfo.setComment(rs.getString(5));
            columnInfoList.add(columnInfo);
        }
        if (columnInfoList.isEmpty()) {
            throw DataXException.asDataXException(AdsWriterErrorCode.NO_ADS_TABLE, table + "不存在或者查询不到列信息. ");
        }
        tableInfo.setColumns(columnInfoList);
        tableInfo.setTableSchema(schema);
        tableInfo.setTableName(table);
        DBUtil.closeDBResources(rs, statement, null);
        String tableMetaSql = String.format("select update_type, partition_type, partition_column, partition_count, primary_key_columns from information_schema.tables where lower(table_schema) = `'%s'` and lower(table_name) = `'%s'`", schema.toLowerCase(), table.toLowerCase());
        LOG.info(String.format("检查表信息sql语句:%s", tableMetaSql));
        statement = connection.createStatement();
        rs = statement.executeQuery(tableMetaSql);
        while (DBUtil.asyncResultSetNext(rs)) {
            tableInfo.setUpdateType(rs.getString(1));
            tableInfo.setPartitionType(rs.getString(2));
            tableInfo.setPartitionColumn(rs.getString(3));
            tableInfo.setPartitionCount(rs.getInt(4));
            //primary_key_columns  ads主键是逗号分隔的，可以有多个
            String primaryKeyColumns = rs.getString(5);
            if (StringUtils.isNotBlank(primaryKeyColumns)) {
                tableInfo.setPrimaryKeyColumns(Arrays.asList(StringUtils.split(primaryKeyColumns, ",")));
            } else {
                tableInfo.setPrimaryKeyColumns(null);
            }
            break;
        }
        DBUtil.closeDBResources(rs, statement, null);
        return tableInfo;
    } catch (ClassNotFoundException e) {
        throw new AdsException(AdsException.OTHER, e.getMessage(), e);
    } catch (SQLException e) {
        throw new AdsException(AdsException.OTHER, e.getMessage(), e);
    } catch (DataXException e) {
        throw e;
    } catch (Exception e) {
        throw new AdsException(AdsException.OTHER, e.getMessage(), e);
    } finally {
        if (rs != null) {
            try {
                rs.close();
            } catch (SQLException e) {
            // Ignore exception
            }
        }
        if (statement != null) {
            try {
                statement.close();
            } catch (SQLException e) {
            // Ignore exception
            }
        }
        if (connection != null) {
            try {
                connection.close();
            } catch (SQLException e) {
            // Ignore exception
            }
        }
    }
}

Example 4 with AdsException

use of com.alibaba.datax.plugin.writer.adswriter.AdsException in project DataX by alibaba.

the class AdsHelper method loadData.

/**
     * Submit LOAD DATA command.
     * 
     * @param table The target ADS table
     * @param partition The partition option in the form of "(partition_name,...)"
     * @param sourcePath The source path
     * @param overwrite
     * @return
     * @throws AdsException
     */
public String loadData(String table, String partition, String sourcePath, boolean overwrite) throws AdsException {
    if (table == null) {
        throw new AdsException(AdsException.ADS_LOADDATA_TABLE_NULL, "ADS LOAD DATA table is null.", null);
    }
    if (sourcePath == null) {
        throw new AdsException(AdsException.ADS_LOADDATA_SOURCEPATH_NULL, "ADS LOAD DATA source path is null.", null);
    }
    if (adsURL == null) {
        throw new AdsException(AdsException.ADS_CONN_URL_NOT_SET, "ADS JDBC connection URL was not set.", null);
    }
    if (userName == null) {
        throw new AdsException(AdsException.ADS_CONN_USERNAME_NOT_SET, "ADS JDBC connection user name was not set.", null);
    }
    if (password == null) {
        throw new AdsException(AdsException.ADS_CONN_PASSWORD_NOT_SET, "ADS JDBC connection password was not set.", null);
    }
    if (schema == null) {
        throw new AdsException(AdsException.ADS_CONN_SCHEMA_NOT_SET, "ADS JDBC connection schema was not set.", null);
    }
    StringBuilder sb = new StringBuilder();
    sb.append("LOAD DATA FROM ");
    if (sourcePath.startsWith("'") && sourcePath.endsWith("'")) {
        sb.append(sourcePath);
    } else {
        sb.append("'" + sourcePath + "'");
    }
    if (overwrite) {
        sb.append(" OVERWRITE");
    }
    sb.append(" INTO TABLE ");
    sb.append(schema + "." + table);
    if (partition != null && !partition.trim().equals("")) {
        String partitionTrim = partition.trim();
        if (partitionTrim.startsWith("(") && partitionTrim.endsWith(")")) {
            sb.append(" PARTITION " + partition);
        } else {
            sb.append(" PARTITION " + "(" + partition + ")");
        }
    }
    Connection connection = null;
    Statement statement = null;
    ResultSet rs = null;
    try {
        Class.forName("com.mysql.jdbc.Driver");
        String url = AdsUtil.prepareJdbcUrl(this.adsURL, this.schema, this.socketTimeout, this.suffix);
        Properties connectionProps = new Properties();
        connectionProps.put("user", userName);
        connectionProps.put("password", password);
        connection = DriverManager.getConnection(url, connectionProps);
        statement = connection.createStatement();
        LOG.info("正在从ODPS数据库导数据到ADS中: " + sb.toString());
        LOG.info("由于ADS的限制，ADS导数据最少需要20分钟，请耐心等待");
        rs = statement.executeQuery(sb.toString());
        String jobId = null;
        while (DBUtil.asyncResultSetNext(rs)) {
            jobId = rs.getString(1);
        }
        if (jobId == null) {
            throw new AdsException(AdsException.ADS_LOADDATA_JOBID_NOT_AVAIL, "Job id is not available for the submitted LOAD DATA." + jobId, null);
        }
        return jobId;
    } catch (ClassNotFoundException e) {
        throw new AdsException(AdsException.ADS_LOADDATA_FAILED, e.getMessage(), e);
    } catch (SQLException e) {
        throw new AdsException(AdsException.ADS_LOADDATA_FAILED, e.getMessage(), e);
    } catch (Exception e) {
        throw new AdsException(AdsException.ADS_LOADDATA_FAILED, e.getMessage(), e);
    } finally {
        if (rs != null) {
            try {
                rs.close();
            } catch (SQLException e) {
            // Ignore exception
            }
        }
        if (statement != null) {
            try {
                statement.close();
            } catch (SQLException e) {
            // Ignore exception
            }
        }
        if (connection != null) {
            try {
                connection.close();
            } catch (SQLException e) {
            // Ignore exception
            }
        }
    }
}

Example 5 with AdsException

use of com.alibaba.datax.plugin.writer.adswriter.AdsException in project DataX by alibaba.

the class AdsInsertUtil method getAdsTableColumns.

/*
     * 返回列顺序为ads建表列顺序
     * */
public static List<ColumnInfo> getAdsTableColumns(Configuration conf) {
    AdsHelper adsHelper = AdsUtil.createAdsHelper(conf);
    TableInfo tableInfo = null;
    String adsTable = conf.getString(Key.ADS_TABLE);
    try {
        tableInfo = adsHelper.getTableInfo(adsTable);
    } catch (AdsException e) {
        throw DataXException.asDataXException(AdsWriterErrorCode.GET_ADS_TABLE_MEATA_FAILED, e);
    }
    List<ColumnInfo> columnInfos = tableInfo.getColumns();
    return columnInfos;
}