Examples with RecordReader com.aliyun.odps.data.RecordReader used on opensource projects

Example 1 with RecordReader

use of com.aliyun.odps.data.RecordReader in project DataX by alibaba.

the class ReaderProxy method doRead.

// warn: odps 分区列和正常列不能重名, 所有列都不不区分大小写
public void doRead() {
    try {
        LOG.info("start={}, count={}", start, count);
        //RecordReader recordReader = downloadSession.openRecordReader(start, count, isCompress);
        RecordReader recordReader = OdpsUtil.getRecordReader(downloadSession, start, count, isCompress);
        Record odpsRecord;
        Map<String, String> partitionMap = this.parseCurrentPartitionValue();
        int retryTimes = 1;
        while (true) {
            try {
                odpsRecord = recordReader.read();
            } catch (Exception e) {
                //odps read 异常后重试10次
                LOG.warn("warn : odps read exception: {}", e.getMessage());
                if (retryTimes < 10) {
                    try {
                        Thread.sleep(2000);
                    } catch (InterruptedException ignored) {
                    }
                    recordReader = downloadSession.openRecordReader(start, count, isCompress);
                    LOG.warn("odps-read-exception, 重试第{}次", retryTimes);
                    retryTimes++;
                    continue;
                } else {
                    throw DataXException.asDataXException(OdpsReaderErrorCode.ODPS_READ_EXCEPTION, e);
                }
            }
            //记录已经读取的点
            start++;
            count--;
            if (odpsRecord != null) {
                com.alibaba.datax.common.element.Record dataXRecord = recordSender.createRecord();
                // sets(columnName), always contain
                for (Pair<String, ColumnType> pair : this.parsedColumns) {
                    String columnName = pair.getLeft();
                    switch(pair.getRight()) {
                        case PARTITION:
                            String partitionColumnValue = this.getPartitionColumnValue(partitionMap, columnName);
                            this.odpsColumnToDataXField(odpsRecord, dataXRecord, this.columnTypeMap.get(columnName), partitionColumnValue, true);
                            break;
                        case NORMAL:
                            this.odpsColumnToDataXField(odpsRecord, dataXRecord, this.columnTypeMap.get(columnName), columnName, false);
                            break;
                        case CONSTANT:
                            dataXRecord.addColumn(new StringColumn(columnName));
                            break;
                        default:
                            break;
                    }
                }
                recordSender.sendToWriter(dataXRecord);
            } else {
                break;
            }
        }
        //fixed, 避免recordReader.close失败，跟鸣天确认过，可以不用关闭RecordReader
        try {
            recordReader.close();
        } catch (Exception e) {
            LOG.warn("recordReader close exception", e);
        }
    } catch (DataXException e) {
        throw e;
    } catch (Exception e) {
        // warn: if dirty
        throw DataXException.asDataXException(OdpsReaderErrorCode.READ_DATA_FAIL, e);
    }
}