Examples with DataXException com.alibaba.datax.common.exception.DataXException used on opensource projects

Example 1 with DataXException

use of com.alibaba.datax.common.exception.DataXException in project DataX by alibaba.

the class AdsHelper method getTableInfo.

/**
     * Obtain the table meta information.
     * 
     * @param table The table
     * @return The table meta information
     * @throws com.alibaba.datax.plugin.writer.adswriter.AdsException
     */
public TableInfo getTableInfo(String table) throws AdsException {
    if (table == null) {
        throw new AdsException(AdsException.ADS_TABLEMETA_TABLE_NULL, "Table is null.", null);
    }
    if (adsURL == null) {
        throw new AdsException(AdsException.ADS_CONN_URL_NOT_SET, "ADS JDBC connection URL was not set.", null);
    }
    if (userName == null) {
        throw new AdsException(AdsException.ADS_CONN_USERNAME_NOT_SET, "ADS JDBC connection user name was not set.", null);
    }
    if (password == null) {
        throw new AdsException(AdsException.ADS_CONN_PASSWORD_NOT_SET, "ADS JDBC connection password was not set.", null);
    }
    if (schema == null) {
        throw new AdsException(AdsException.ADS_CONN_SCHEMA_NOT_SET, "ADS JDBC connection schema was not set.", null);
    }
    Connection connection = null;
    Statement statement = null;
    ResultSet rs = null;
    try {
        Class.forName("com.mysql.jdbc.Driver");
        String url = AdsUtil.prepareJdbcUrl(this.adsURL, this.schema, this.socketTimeout, this.suffix);
        Properties connectionProps = new Properties();
        connectionProps.put("user", userName);
        connectionProps.put("password", password);
        connection = DriverManager.getConnection(url, connectionProps);
        statement = connection.createStatement();
        // ads 表名、schema名不区分大小写, 提高用户易用性, 注意列顺序性
        String columnMetaSql = String.format("select ordinal_position,column_name,data_type,type_name,column_comment from information_schema.columns where lower(table_schema) = `'%s'` and lower(table_name) = `'%s'` order by ordinal_position", schema.toLowerCase(), table.toLowerCase());
        LOG.info(String.format("检查列信息sql语句:%s", columnMetaSql));
        rs = statement.executeQuery(columnMetaSql);
        TableInfo tableInfo = new TableInfo();
        List<ColumnInfo> columnInfoList = new ArrayList<ColumnInfo>();
        while (DBUtil.asyncResultSetNext(rs)) {
            ColumnInfo columnInfo = new ColumnInfo();
            columnInfo.setOrdinal(rs.getInt(1));
            columnInfo.setName(rs.getString(2));
            //columnInfo.setDataType(ColumnDataType.getDataType(rs.getInt(3))); //for ads version < 0.7
            //columnInfo.setDataType(ColumnDataType.getTypeByName(rs.getString(3).toUpperCase())); //for ads version 0.8
            //for ads version 0.8 & 0.7
            columnInfo.setDataType(ColumnDataType.getTypeByName(rs.getString(4).toUpperCase()));
            columnInfo.setComment(rs.getString(5));
            columnInfoList.add(columnInfo);
        }
        if (columnInfoList.isEmpty()) {
            throw DataXException.asDataXException(AdsWriterErrorCode.NO_ADS_TABLE, table + "不存在或者查询不到列信息. ");
        }
        tableInfo.setColumns(columnInfoList);
        tableInfo.setTableSchema(schema);
        tableInfo.setTableName(table);
        DBUtil.closeDBResources(rs, statement, null);
        String tableMetaSql = String.format("select update_type, partition_type, partition_column, partition_count, primary_key_columns from information_schema.tables where lower(table_schema) = `'%s'` and lower(table_name) = `'%s'`", schema.toLowerCase(), table.toLowerCase());
        LOG.info(String.format("检查表信息sql语句:%s", tableMetaSql));
        statement = connection.createStatement();
        rs = statement.executeQuery(tableMetaSql);
        while (DBUtil.asyncResultSetNext(rs)) {
            tableInfo.setUpdateType(rs.getString(1));
            tableInfo.setPartitionType(rs.getString(2));
            tableInfo.setPartitionColumn(rs.getString(3));
            tableInfo.setPartitionCount(rs.getInt(4));
            //primary_key_columns  ads主键是逗号分隔的，可以有多个
            String primaryKeyColumns = rs.getString(5);
            if (StringUtils.isNotBlank(primaryKeyColumns)) {
                tableInfo.setPrimaryKeyColumns(Arrays.asList(StringUtils.split(primaryKeyColumns, ",")));
            } else {
                tableInfo.setPrimaryKeyColumns(null);
            }
            break;
        }
        DBUtil.closeDBResources(rs, statement, null);
        return tableInfo;
    } catch (ClassNotFoundException e) {
        throw new AdsException(AdsException.OTHER, e.getMessage(), e);
    } catch (SQLException e) {
        throw new AdsException(AdsException.OTHER, e.getMessage(), e);
    } catch (DataXException e) {
        throw e;
    } catch (Exception e) {
        throw new AdsException(AdsException.OTHER, e.getMessage(), e);
    } finally {
        if (rs != null) {
            try {
                rs.close();
            } catch (SQLException e) {
            // Ignore exception
            }
        }
        if (statement != null) {
            try {
                statement.close();
            } catch (SQLException e) {
            // Ignore exception
            }
        }
        if (connection != null) {
            try {
                connection.close();
            } catch (SQLException e) {
            // Ignore exception
            }
        }
    }
}

Example 2 with DataXException

use of com.alibaba.datax.common.exception.DataXException in project DataX by alibaba.

the class HttpClientUtilTest method testExecuteAndGetWithRetry2.

/**
     * 和测试方法一样：testExecuteAndGetWithRetry()，只是换了一种写法，直接采用 Mockito 进行验证的。
     */
@Test
public void testExecuteAndGetWithRetry2() throws Exception {
    String url = "http://127.0.0.1/:8080";
    HttpRequestBase httpRequestBase = new HttpGet(url);
    HttpClientUtil httpClientUtil = Mockito.spy(HttpClientUtil.getHttpClientUtil());
    Mockito.doThrow(new Exception("one")).doThrow(new Exception("two")).doThrow(new Exception("three")).doReturn("成功").when(httpClientUtil).executeAndGet(httpRequestBase);
    String str = httpClientUtil.executeAndGetWithRetry(httpRequestBase, 4, 1000l);
    Assert.assertEquals(str, "成功");
    Mockito.reset(httpClientUtil);
    Mockito.doThrow(new Exception("one")).doThrow(new Exception("two")).doThrow(new Exception("three")).doReturn("成功").when(httpClientUtil).executeAndGet(httpRequestBase);
    try {
        httpClientUtil.executeAndGetWithRetry(httpRequestBase, 2, 1000l);
    } catch (Exception e) {
        Assert.assertTrue(e instanceof DataXException);
        Assert.assertTrue(e.getMessage().contains("two"));
    }
    httpClientUtil.destroy();
}

Example 3 with DataXException

use of com.alibaba.datax.common.exception.DataXException in project DataX by alibaba.

the class HttpClientUtilTest method testExecuteAndGetWithRetry.

@Test
public void testExecuteAndGetWithRetry() throws Exception {
    String url = "http://127.0.0.1/:8080";
    HttpRequestBase httpRequestBase = new HttpGet(url);
    HttpClientUtil httpClientUtil = PowerMockito.spy(HttpClientUtil.getHttpClientUtil());
    PowerMockito.doAnswer(new Answer<Object>() {

        @Override
        public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
            System.out.println("失败第1次");
            return new Exception("失败第1次");
        }
    }).doAnswer(new Answer<Object>() {

        @Override
        public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
            System.out.println("失败第2次");
            return new Exception("失败第2次");
        }
    }).doAnswer(new Answer<Object>() {

        @Override
        public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
            System.out.println("失败第3次");
            return new Exception("失败第3次");
        }
    }).doAnswer(new Answer<Object>() {

        @Override
        public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
            System.out.println("失败第4次");
            return new Exception("失败第4次");
        }
    }).doReturn("成功").when(httpClientUtil).executeAndGet(any(HttpRequestBase.class));
    String str = httpClientUtil.executeAndGetWithRetry(httpRequestBase, 5, 1000l);
    Assert.assertEquals(str, "成功");
    try {
        httpClientUtil.executeAndGetWithRetry(httpRequestBase, 2, 1000l);
    } catch (Exception e) {
        Assert.assertTrue(e instanceof DataXException);
    }
    httpClientUtil.destroy();
}

Example 4 with DataXException

use of com.alibaba.datax.common.exception.DataXException in project DataX by alibaba.

the class ReaderProxy method doRead.

// warn: odps 分区列和正常列不能重名, 所有列都不不区分大小写
public void doRead() {
    try {
        LOG.info("start={}, count={}", start, count);
        //RecordReader recordReader = downloadSession.openRecordReader(start, count, isCompress);
        RecordReader recordReader = OdpsUtil.getRecordReader(downloadSession, start, count, isCompress);
        Record odpsRecord;
        Map<String, String> partitionMap = this.parseCurrentPartitionValue();
        int retryTimes = 1;
        while (true) {
            try {
                odpsRecord = recordReader.read();
            } catch (Exception e) {
                //odps read 异常后重试10次
                LOG.warn("warn : odps read exception: {}", e.getMessage());
                if (retryTimes < 10) {
                    try {
                        Thread.sleep(2000);
                    } catch (InterruptedException ignored) {
                    }
                    recordReader = downloadSession.openRecordReader(start, count, isCompress);
                    LOG.warn("odps-read-exception, 重试第{}次", retryTimes);
                    retryTimes++;
                    continue;
                } else {
                    throw DataXException.asDataXException(OdpsReaderErrorCode.ODPS_READ_EXCEPTION, e);
                }
            }
            //记录已经读取的点
            start++;
            count--;
            if (odpsRecord != null) {
                com.alibaba.datax.common.element.Record dataXRecord = recordSender.createRecord();
                // sets(columnName), always contain
                for (Pair<String, ColumnType> pair : this.parsedColumns) {
                    String columnName = pair.getLeft();
                    switch(pair.getRight()) {
                        case PARTITION:
                            String partitionColumnValue = this.getPartitionColumnValue(partitionMap, columnName);
                            this.odpsColumnToDataXField(odpsRecord, dataXRecord, this.columnTypeMap.get(columnName), partitionColumnValue, true);
                            break;
                        case NORMAL:
                            this.odpsColumnToDataXField(odpsRecord, dataXRecord, this.columnTypeMap.get(columnName), columnName, false);
                            break;
                        case CONSTANT:
                            dataXRecord.addColumn(new StringColumn(columnName));
                            break;
                        default:
                            break;
                    }
                }
                recordSender.sendToWriter(dataXRecord);
            } else {
                break;
            }
        }
        //fixed, 避免recordReader.close失败，跟鸣天确认过，可以不用关闭RecordReader
        try {
            recordReader.close();
        } catch (Exception e) {
            LOG.warn("recordReader close exception", e);
        }
    } catch (DataXException e) {
        throw e;
    } catch (Exception e) {
        // warn: if dirty
        throw DataXException.asDataXException(OdpsReaderErrorCode.READ_DATA_FAIL, e);
    }
}

Example 5 with DataXException

use of com.alibaba.datax.common.exception.DataXException in project DataX by alibaba.

the class UnstructuredStorageReaderUtil method transportOneRecord.

public static Record transportOneRecord(RecordSender recordSender, List<ColumnEntry> columnConfigs, String[] sourceLine, String nullFormat, TaskPluginCollector taskPluginCollector) {
    Record record = recordSender.createRecord();
    Column columnGenerated = null;
    // 创建都为String类型column的record
    if (null == columnConfigs || columnConfigs.size() == 0) {
        for (String columnValue : sourceLine) {
            // not equalsIgnoreCase, it's all ok if nullFormat is null
            if (columnValue.equals(nullFormat)) {
                columnGenerated = new StringColumn(null);
            } else {
                columnGenerated = new StringColumn(columnValue);
            }
            record.addColumn(columnGenerated);
        }
        recordSender.sendToWriter(record);
    } else {
        try {
            for (ColumnEntry columnConfig : columnConfigs) {
                String columnType = columnConfig.getType();
                Integer columnIndex = columnConfig.getIndex();
                String columnConst = columnConfig.getValue();
                String columnValue = null;
                if (null == columnIndex && null == columnConst) {
                    throw DataXException.asDataXException(UnstructuredStorageReaderErrorCode.NO_INDEX_VALUE, "由于您配置了type, 则至少需要配置 index 或 value");
                }
                if (null != columnIndex && null != columnConst) {
                    throw DataXException.asDataXException(UnstructuredStorageReaderErrorCode.MIXED_INDEX_VALUE, "您混合配置了index, value, 每一列同时仅能选择其中一种");
                }
                if (null != columnIndex) {
                    if (columnIndex >= sourceLine.length) {
                        String message = String.format("您尝试读取的列越界,源文件该行有 [%s] 列,您尝试读取第 [%s] 列, 数据详情[%s]", sourceLine.length, columnIndex + 1, StringUtils.join(sourceLine, ","));
                        LOG.warn(message);
                        throw new IndexOutOfBoundsException(message);
                    }
                    columnValue = sourceLine[columnIndex];
                } else {
                    columnValue = columnConst;
                }
                Type type = Type.valueOf(columnType.toUpperCase());
                // it's all ok if nullFormat is null
                if (columnValue.equals(nullFormat)) {
                    columnValue = null;
                }
                switch(type) {
                    case STRING:
                        columnGenerated = new StringColumn(columnValue);
                        break;
                    case LONG:
                        try {
                            columnGenerated = new LongColumn(columnValue);
                        } catch (Exception e) {
                            throw new IllegalArgumentException(String.format("类型转换错误, 无法将[%s] 转换为[%s]", columnValue, "LONG"));
                        }
                        break;
                    case DOUBLE:
                        try {
                            columnGenerated = new DoubleColumn(columnValue);
                        } catch (Exception e) {
                            throw new IllegalArgumentException(String.format("类型转换错误, 无法将[%s] 转换为[%s]", columnValue, "DOUBLE"));
                        }
                        break;
                    case BOOLEAN:
                        try {
                            columnGenerated = new BoolColumn(columnValue);
                        } catch (Exception e) {
                            throw new IllegalArgumentException(String.format("类型转换错误, 无法将[%s] 转换为[%s]", columnValue, "BOOLEAN"));
                        }
                        break;
                    case DATE:
                        try {
                            if (columnValue == null) {
                                Date date = null;
                                columnGenerated = new DateColumn(date);
                            } else {
                                String formatString = columnConfig.getFormat();
                                //if (null != formatString) {
                                if (StringUtils.isNotBlank(formatString)) {
                                    // 用户自己配置的格式转换, 脏数据行为出现变化
                                    DateFormat format = columnConfig.getDateFormat();
                                    columnGenerated = new DateColumn(format.parse(columnValue));
                                } else {
                                    // 框架尝试转换
                                    columnGenerated = new DateColumn(new StringColumn(columnValue).asDate());
                                }
                            }
                        } catch (Exception e) {
                            throw new IllegalArgumentException(String.format("类型转换错误, 无法将[%s] 转换为[%s]", columnValue, "DATE"));
                        }
                        break;
                    default:
                        String errorMessage = String.format("您配置的列类型暂不支持 : [%s]", columnType);
                        LOG.error(errorMessage);
                        throw DataXException.asDataXException(UnstructuredStorageReaderErrorCode.NOT_SUPPORT_TYPE, errorMessage);
                }
                record.addColumn(columnGenerated);
            }
            recordSender.sendToWriter(record);
        } catch (IllegalArgumentException iae) {
            taskPluginCollector.collectDirtyRecord(record, iae.getMessage());
        } catch (IndexOutOfBoundsException ioe) {
            taskPluginCollector.collectDirtyRecord(record, ioe.getMessage());
        } catch (Exception e) {
            if (e instanceof DataXException) {
                throw (DataXException) e;
            }
            // 每一种转换失败都是脏数据处理,包括数字格式 & 日期格式
            taskPluginCollector.collectDirtyRecord(record, e.getMessage());
        }
    }
    return record;
}