Examples with Record com.alibaba.datax.common.element.Record used on opensource projects

Example 1 with Record

use of com.alibaba.datax.common.element.Record in project DataX by alibaba.

the class AdsInsertProxy method doBatchRecordDml.

//warn: ADS 无法支持事物roll back都是不管用
@SuppressWarnings("resource")
private void doBatchRecordDml(List<Record> buffer, String mode) throws Exception {
    Statement statement = null;
    String sql = null;
    try {
        int bufferSize = buffer.size();
        if (buffer.isEmpty()) {
            return;
        }
        StringBuilder sqlSb = new StringBuilder();
        // connection.setAutoCommit(true);
        //mysql impl warn: if a database access error occurs or this method is called on a closed connection throw SQLException
        statement = this.currentConnection.createStatement();
        sqlSb.append(this.generateDmlSql(this.currentConnection, buffer.get(0), mode));
        for (int i = 1; i < bufferSize; i++) {
            Record record = buffer.get(i);
            this.appendDmlSqlValues(this.currentConnection, record, sqlSb, mode);
        }
        sql = sqlSb.toString();
        if (IS_DEBUG_ENABLE) {
            LOG.debug(sql);
        }
        @SuppressWarnings("unused") int status = statement.executeUpdate(sql);
        sql = null;
    } catch (SQLException e) {
        LOG.warn("doBatchRecordDml meet a exception: " + sql, e);
        Exception eachException = e;
        // 避免死循环
        int maxIter = 0;
        while (null != eachException && maxIter < AdsInsertProxy.MAX_EXCEPTION_CAUSE_ITER) {
            if (this.isRetryable(eachException)) {
                LOG.warn("doBatchRecordDml meet a retry exception: " + e.getMessage());
                this.currentConnection = AdsUtil.getAdsConnect(this.configuration);
                throw eachException;
            } else {
                try {
                    Throwable causeThrowable = eachException.getCause();
                    eachException = causeThrowable == null ? null : (Exception) causeThrowable;
                } catch (Exception castException) {
                    LOG.warn("doBatchRecordDml meet a no! retry exception: " + e.getMessage());
                    throw e;
                }
            }
            maxIter++;
        }
        throw e;
    } catch (Exception e) {
        LOG.error("插入异常, sql: " + sql);
        throw DataXException.asDataXException(DBUtilErrorCode.WRITE_DATA_ERROR, e);
    } finally {
        DBUtil.closeDBResources(statement, null);
    }
}

Example 2 with Record

use of com.alibaba.datax.common.element.Record in project DataX by alibaba.

the class BufferedRecordExchanger method getFromReader.

@Override
public Record getFromReader() {
    if (shutdown) {
        throw DataXException.asDataXException(CommonErrorCode.SHUT_DOWN_TASK, "");
    }
    boolean isEmpty = (this.bufferIndex >= this.buffer.size());
    if (isEmpty) {
        receive();
    }
    Record record = this.buffer.get(this.bufferIndex++);
    if (record instanceof TerminateRecord) {
        record = null;
    }
    return record;
}

Example 3 with Record

use of com.alibaba.datax.common.element.Record in project DataX by alibaba.

the class BufferedRecordTransformerExchanger method getFromReader.

@Override
public Record getFromReader() {
    if (shutdown) {
        throw DataXException.asDataXException(CommonErrorCode.SHUT_DOWN_TASK, "");
    }
    boolean isEmpty = (this.bufferIndex >= this.buffer.size());
    if (isEmpty) {
        receive();
    }
    Record record = this.buffer.get(this.bufferIndex++);
    if (record instanceof TerminateRecord) {
        record = null;
    }
    return record;
}

Example 4 with Record

use of com.alibaba.datax.common.element.Record in project DataX by alibaba.

the class Consumer method test_BlockAndSeq.

@Test
public void test_BlockAndSeq() throws InterruptedException {
    int tryCount = 100;
    int capacity = ConfigurationProducer.produce().getInt(CoreConstant.DATAX_CORE_TRANSPORT_CHANNEL_CAPACITY);
    System.out.println("capacity: " + capacity);
    Thread thread = new Thread(new Consumer(this.channel, tryCount * capacity));
    thread.start();
    List<Record> records = new ArrayList<Record>(capacity);
    for (int i = 0; i < capacity; i++) {
        Record record = RecordProducer.produceRecord();
        record.setColumn(0, new LongColumn(i));
        records.add(record);
    }
    for (int i = 0; i < tryCount; i++) {
        this.channel.pushAll(records);
    }
    Thread.sleep(5000L);
    this.channel.push(TerminateRecord.get());
    Thread.sleep(1000L);
    thread.join();
}

Example 5 with Record

use of com.alibaba.datax.common.element.Record in project DataX by alibaba.

the class RecordExchangerTest method test_BufferExchanger.

@Test
public void test_BufferExchanger() {
    Configuration configuration = ConfigurationProducer.produce();
    configuration.set(CoreConstant.DATAX_CORE_CONTAINER_TASKGROUP_ID, 1);
    Channel channel = new MemoryChannel(configuration);
    channel.setCommunication(new Communication());
    TaskPluginCollector pluginCollector = mock(TaskPluginCollector.class);
    int capacity = 10;
    Record record = null;
    BufferedRecordExchanger recordExchanger = new BufferedRecordExchanger(channel, pluginCollector);
    for (int i = 0; i < capacity; i++) {
        record = RecordProducer.produceRecord();
        record.setColumn(0, new LongColumn(i));
        recordExchanger.sendToWriter(record);
    }
    recordExchanger.flush();
    channel.close();
    int counter = 0;
    while ((record = recordExchanger.getFromReader()) != null) {
        System.out.println(record.getColumn(0).toString());
        Assert.assertTrue(record.getColumn(0).asLong() == counter);
        counter++;
    }
    System.out.println(String.format("Capacity: %d Counter: %d .", capacity, counter));
    Assert.assertTrue(capacity == counter);
}