Examples with RowBatch com.alibaba.otter.shared.etl.model.RowBatch used on opensource projects

Example 21 with RowBatch

use of com.alibaba.otter.shared.etl.model.RowBatch in project otter by alibaba.

the class FreedomExtractorTest method test_oracle.

@Test
public void test_oracle() {
    final Pipeline pipeline = new Pipeline();
    pipeline.setId(100L);
    int start = RandomUtils.nextInt();
    int count = 10;
    List<DataMediaPair> pairs = getDataMediaPairForOracle(start, count);
    pipeline.setPairs(pairs);
    new NonStrictExpectations() {

        {
            configClientService.findPipeline(100L);
            returns(pipeline);
        }
    };
    // 构造数据
    RowBatch rowBatch = new RowBatch();
    rowBatch.setIdentity(identity);
    for (int tableId = start; tableId < start + count; tableId++) {
        for (int i = start; i < start + count; i++) {
            EventData eventData = getEventData(tableId, i);
            eventData.setSchemaName("retl");
            eventData.setTableName("retl_buffer");
            rowBatch.merge(eventData);
        }
    }
    DbBatch dbBatch = new DbBatch(rowBatch);
    freedomExtractor.extract(dbBatch);
    want.collection(dbBatch.getRowBatch().getDatas()).sizeEq(count * count);
}

Example 22 with RowBatch

use of com.alibaba.otter.shared.etl.model.RowBatch in project otter by alibaba.

the class DbLoadActionTest method test_db_load_oracle.

@Test
public void test_db_load_oracle() {
    ArbitrateConfigRegistry.regist(configClientService);
    dbLoadAction = (DbLoadAction) TestedObject.getSpringBeanFactory().getBean("dbLoadAction");
    final Channel channel = new Channel();
    channel.setId(1L);
    final Pipeline pipeline = new Pipeline();
    pipeline.setId(100L);
    List<DataMediaPair> pairs = generatorDataMediaPairForOracle(20);
    pipeline.setPairs(pairs);
    pipeline.getParameters().merge(new SystemParameter());
    pipeline.getParameters().merge(new ChannelParameter());
    // final Pipeline oppositePipeline = new Pipeline();
    // oppositePipeline.setId(101L);
    channel.setPipelines(Arrays.asList(pipeline));
    final Node currentNode = new Node();
    currentNode.setId(1L);
    new NonStrictExpectations() {

        {
            configClientService.findChannel(anyLong);
            returns(channel);
            configClientService.findPipeline(anyLong);
            returns(pipeline);
            configClientService.currentNode();
            returns(currentNode);
        }
    };
    Identity identity = new Identity();
    identity.setChannelId(100L);
    identity.setPipelineId(100L);
    identity.setProcessId(100L);
    RowBatch rowBatch = new RowBatch();
    rowBatch.setIdentity(identity);
    List<EventData> eventDatas = generatorEventDataForOracle(0, 20, EventType.INSERT);
    for (EventData eventData : eventDatas) {
        rowBatch.merge(eventData);
    }
    eventDatas = generatorEventDataForOracle(10, 10, EventType.INSERT);
    for (EventData eventData : eventDatas) {
        rowBatch.merge(eventData);
    }
    eventDatas = generatorEventDataForOracle(19, 1, EventType.DELETE);
    for (EventData eventData : eventDatas) {
        rowBatch.merge(eventData);
    }
    WeightController controller = new WeightController(1);
    dbLoadAction.load(rowBatch, controller);
}

Example 23 with RowBatch

use of com.alibaba.otter.shared.etl.model.RowBatch in project otter by alibaba.

the class OtterTransformerFactory method merge.

// =============================== helper method
// ============================
// 将生成的item对象合并到结果对象中
private synchronized void merge(Identity identity, Map<Class, BatchObject> data, Object item) {
    Class clazz = item.getClass();
    BatchObject batchObject = data.get(clazz);
    // 初始化一下对象
    if (batchObject == null) {
        batchObject = initBatchObject(identity, clazz);
        data.put(clazz, batchObject);
    }
    // 进行merge处理
    if (batchObject instanceof RowBatch) {
        ((RowBatch) batchObject).merge((EventData) item);
    } else if (batchObject instanceof FileBatch) {
        ((FileBatch) batchObject).getFiles().add((FileData) item);
    } else {
        throw new TransformException("no support Data[" + clazz.getName() + "]");
    }
}