Examples with WriterConfig com.aliyun.openservices.ots.internal.writer.WriterConfig used on opensource projects

Example 1 with WriterConfig

use of com.aliyun.openservices.ots.internal.writer.WriterConfig in project DataX by alibaba.

the class OtsWriterSlaveProxy method write.

public void write(RecordReceiver recordReceiver, TaskPluginCollector collector) throws Exception {
    LOG.info("Writer slave started.");
    WriterConfig writerConfig = new WriterConfig();
    writerConfig.setConcurrency(conf.getConcurrencyWrite());
    writerConfig.setMaxBatchRowsCount(conf.getBatchWriteCount());
    writerConfig.setMaxBatchSize(conf.getRestrictConf().getRequestTotalSizeLimition());
    writerConfig.setBufferSize(conf.getBufferSize());
    writerConfig.setMaxAttrColumnSize(conf.getRestrictConf().getAttributeColumnSize());
    writerConfig.setMaxColumnsCount(conf.getRestrictConf().getMaxColumnsCount());
    writerConfig.setMaxPKColumnSize(conf.getRestrictConf().getPrimaryKeyColumnSize());
    otsWriter = new DefaultOTSWriter(otsAsync, conf.getTableName(), writerConfig, new WriterCallback(collector), Executors.newFixedThreadPool(3));
    int expectColumnCount = conf.getPrimaryKeyColumn().size() + conf.getAttributeColumn().size();
    Record record;
    while ((record = recordReceiver.getFromReader()) != null) {
        LOG.debug("Record Raw: {}", record.toString());
        int columnCount = record.getColumnNumber();
        if (columnCount != expectColumnCount) {
            // 如果Column的个数和预期的个数不一致时，认为是系统故障或者用户配置Column错误，异常退出
            throw new IllegalArgumentException(String.format(OTSErrorMessage.RECORD_AND_COLUMN_SIZE_ERROR, columnCount, expectColumnCount));
        }
        // 类型转换
        try {
            RowPrimaryKey primaryKey = Common.getPKFromRecord(conf.getPrimaryKeyColumn(), record);
            List<Pair<String, ColumnValue>> attributes = Common.getAttrFromRecord(conf.getPrimaryKeyColumn().size(), conf.getAttributeColumn(), record);
            RowChange rowChange = Common.columnValuesToRowChange(conf.getTableName(), conf.getOperation(), primaryKey, attributes);
            WithRecord withRecord = (WithRecord) rowChange;
            withRecord.setRecord(record);
            otsWriter.addRowChange(rowChange);
        } catch (IllegalArgumentException e) {
            LOG.warn("Found dirty data.", e);
            collector.collectDirtyRecord(record, e.getMessage());
        } catch (ClientException e) {
            LOG.warn("Found dirty data.", e);
            collector.collectDirtyRecord(record, e.getMessage());
        }
    }
    otsWriter.close();
    LOG.info("Writer slave finished.");
}