Examples with RowBatch com.alibaba.otter.shared.etl.model.RowBatch used on opensource projects

Example 6 with RowBatch

use of com.alibaba.otter.shared.etl.model.RowBatch in project otter by alibaba.

the class OtterLoaderFactoryIntegration method test_simple.

@Test
public void test_simple() {
    Identity identity = new Identity();
    identity.setChannelId(100L);
    identity.setPipelineId(100L);
    identity.setProcessId(100L);
    RowBatch rowBatch = new RowBatch();
    rowBatch.setIdentity(identity);
    FileBatch fileBatch = new FileBatch();
    fileBatch.setIdentity(identity);
    final DbBatch dbBatch = new DbBatch();
    dbBatch.setRowBatch(rowBatch);
    dbBatch.setFileBatch(fileBatch);
    final CountDownLatch latch = new CountDownLatch(1);
    executorService.submit(new Runnable() {

        public void run() {
            System.out.println("first run!!!!!!");
            otterLoaderFactory.load(dbBatch);
            latch.countDown();
        }
    });
    try {
        latch.await();
    } catch (InterruptedException e) {
    }
}

Example 7 with RowBatch

use of com.alibaba.otter.shared.etl.model.RowBatch in project otter by alibaba.

the class DbLoadActionTest method test_db_load_mysql.

@Test
public void test_db_load_mysql() {
    ArbitrateConfigRegistry.regist(configClientService);
    dbLoadAction = (DbLoadAction) TestedObject.getSpringBeanFactory().getBean("dbLoadAction");
    final Channel channel = new Channel();
    channel.setId(1L);
    final Pipeline pipeline = new Pipeline();
    pipeline.setId(100L);
    List<DataMediaPair> pairs = generatorDataMediaPairForMysql(20);
    pipeline.setPairs(pairs);
    pipeline.getParameters().merge(new SystemParameter());
    pipeline.getParameters().merge(new ChannelParameter());
    // pipeline.getParameters().setChannelInfo("LJH_DEMO");
    // final Pipeline oppositePipeline = new Pipeline();
    // oppositePipeline.setId(101L);
    channel.setPipelines(Arrays.asList(pipeline));
    final Node currentNode = new Node();
    currentNode.setId(1L);
    new NonStrictExpectations() {

        {
            configClientService.findChannel(anyLong);
            returns(channel);
            configClientService.findPipeline(anyLong);
            returns(pipeline);
            configClientService.currentNode();
            returns(currentNode);
        }
    };
    Identity identity = new Identity();
    identity.setChannelId(100L);
    identity.setPipelineId(100L);
    identity.setProcessId(100L);
    RowBatch rowBatch = new RowBatch();
    rowBatch.setIdentity(identity);
    List<EventData> eventDatas = generatorEventDataForMysql(0, 20, EventType.INSERT);
    for (EventData eventData : eventDatas) {
        rowBatch.merge(eventData);
    }
    eventDatas = generatorEventDataForMysql(10, 10, EventType.INSERT);
    for (EventData eventData : eventDatas) {
        rowBatch.merge(eventData);
    }
    eventDatas = generatorEventDataForMysql(19, 1, EventType.DELETE);
    for (EventData eventData : eventDatas) {
        rowBatch.merge(eventData);
    }
    WeightController controller = new WeightController(1);
    dbLoadAction.load(rowBatch, controller);
}

Example 8 with RowBatch

use of com.alibaba.otter.shared.etl.model.RowBatch in project otter by alibaba.

the class RowDataHttpPipe method saveDbBatch.

// ======================== help method ===================
// 保存对应的dbBatch
private HttpPipeKey saveDbBatch(DbBatch dbBatch) {
    RowBatch rowBatch = dbBatch.getRowBatch();
    // 转化为proto对象
    BatchProto.RowBatch.Builder rowBatchBuilder = BatchProto.RowBatch.newBuilder();
    rowBatchBuilder.setIdentity(build(rowBatch.getIdentity()));
    // 处理具体的字段rowData
    for (EventData eventData : rowBatch.getDatas()) {
        BatchProto.RowData.Builder rowDataBuilder = BatchProto.RowData.newBuilder();
        rowDataBuilder.setPairId(eventData.getPairId());
        rowDataBuilder.setTableId(eventData.getTableId());
        if (eventData.getSchemaName() != null) {
            rowDataBuilder.setSchemaName(eventData.getSchemaName());
        }
        rowDataBuilder.setTableName(eventData.getTableName());
        rowDataBuilder.setEventType(eventData.getEventType().getValue());
        rowDataBuilder.setExecuteTime(eventData.getExecuteTime());
        // add by ljh at 2012-10-31
        if (eventData.getSyncMode() != null) {
            rowDataBuilder.setSyncMode(eventData.getSyncMode().getValue());
        }
        if (eventData.getSyncConsistency() != null) {
            rowDataBuilder.setSyncConsistency(eventData.getSyncConsistency().getValue());
        }
        // 构造key column
        for (EventColumn keyColumn : eventData.getKeys()) {
            rowDataBuilder.addKeys(buildColumn(keyColumn));
        }
        // 构造old key column
        if (CollectionUtils.isEmpty(eventData.getOldKeys()) == false) {
            for (EventColumn keyColumn : eventData.getOldKeys()) {
                rowDataBuilder.addOldKeys(buildColumn(keyColumn));
            }
        }
        // 构造其他 column
        for (EventColumn column : eventData.getColumns()) {
            rowDataBuilder.addColumns(buildColumn(column));
        }
        rowDataBuilder.setRemedy(eventData.isRemedy());
        rowDataBuilder.setSize(eventData.getSize());
        if (StringUtils.isNotEmpty(eventData.getSql())) {
            rowDataBuilder.setSql(eventData.getSql());
        }
        if (StringUtils.isNotEmpty(eventData.getDdlSchemaName())) {
            rowDataBuilder.setDdlSchemaName(eventData.getDdlSchemaName());
        }
        if (StringUtils.isNotEmpty(eventData.getHint())) {
            rowDataBuilder.setHint(eventData.getHint());
        }
        rowDataBuilder.setWithoutSchema(eventData.isWithoutSchema());
        // 添加一条rowData记录
        rowBatchBuilder.addRows(rowDataBuilder.build());
    }
    // 处理下FileBatch
    FileBatch fileBatch = dbBatch.getFileBatch();
    BatchProto.FileBatch.Builder fileBatchBuilder = null;
    fileBatchBuilder = BatchProto.FileBatch.newBuilder();
    fileBatchBuilder.setIdentity(build(fileBatch.getIdentity()));
    // 构造对应的proto对象
    for (FileData fileData : fileBatch.getFiles()) {
        BatchProto.FileData.Builder fileDataBuilder = BatchProto.FileData.newBuilder();
        fileDataBuilder.setPairId(fileData.getPairId());
        fileDataBuilder.setTableId(fileData.getTableId());
        if (fileData.getNameSpace() != null) {
            fileDataBuilder.setNamespace(fileData.getNameSpace());
        }
        if (fileData.getPath() != null) {
            fileDataBuilder.setPath(fileData.getPath());
        }
        fileDataBuilder.setEventType(fileData.getEventType().getValue());
        fileDataBuilder.setSize(fileData.getSize());
        fileDataBuilder.setLastModifiedTime(fileData.getLastModifiedTime());
        // 添加一条fileData记录
        fileBatchBuilder.addFiles(fileDataBuilder.build());
    }
    // 处理构造对应的文件url
    String filename = buildFileName(rowBatch.getIdentity(), ClassUtils.getShortClassName(dbBatch.getClass()));
    // 写入数据
    File file = new File(htdocsDir, filename);
    OutputStream output = null;
    try {
        output = new BufferedOutputStream(new FileOutputStream(file));
        com.alibaba.otter.node.etl.model.protobuf.BatchProto.RowBatch rowBatchProto = rowBatchBuilder.build();
        // 输出大小
        output.write(ByteUtils.int2bytes(rowBatchProto.getSerializedSize()));
        // 输出row batch
        rowBatchProto.writeTo(output);
        com.alibaba.otter.node.etl.model.protobuf.BatchProto.FileBatch fileBatchProto = fileBatchBuilder.build();
        // 输出大小
        output.write(ByteUtils.int2bytes(fileBatchProto.getSerializedSize()));
        // 输出file batch
        fileBatchProto.writeTo(output);
        output.flush();
    } catch (IOException e) {
        throw new PipeException("write_byte_error", e);
    } finally {
        IOUtils.closeQuietly(output);
    }
    HttpPipeKey key = new HttpPipeKey();
    key.setUrl(remoteUrlBuilder.getUrl(rowBatch.getIdentity().getPipelineId(), filename));
    key.setDataType(PipeDataType.DB_BATCH);
    key.setIdentity(rowBatch.getIdentity());
    Pipeline pipeline = configClientService.findPipeline(rowBatch.getIdentity().getPipelineId());
    if (pipeline.getParameters().getUseFileEncrypt()) {
        // 加密处理
        EncryptedData encryptedData = encryptFile(file);
        key.setKey(encryptedData.getKey());
        key.setCrc(encryptedData.getCrc());
    }
    return key;
}

Example 9 with RowBatch

use of com.alibaba.otter.shared.etl.model.RowBatch in project otter by alibaba.

the class DataBatchLoader method split.

/**
     * 将rowBatch中的记录，按找载入的目标数据源进行分类
     */
private List<RowBatch> split(RowBatch rowBatch) {
    final Identity identity = rowBatch.getIdentity();
    Map<DataMediaSource, RowBatch> result = new MapMaker().makeComputingMap(new Function<DataMediaSource, RowBatch>() {

        public RowBatch apply(DataMediaSource input) {
            RowBatch rowBatch = new RowBatch();
            rowBatch.setIdentity(identity);
            return rowBatch;
        }
    });
    for (EventData eventData : rowBatch.getDatas()) {
        // 获取介质信息
        DataMedia media = ConfigHelper.findDataMedia(configClientService.findPipeline(identity.getPipelineId()), eventData.getTableId());
        // 归类
        result.get(media.getSource()).merge(eventData);
    }
    return new ArrayList<RowBatch>(result.values());
}

Example 10 with RowBatch

use of com.alibaba.otter.shared.etl.model.RowBatch in project otter by alibaba.

the class ProcessorExtractor method extract.

public void extract(DbBatch param) throws ExtractException {
    ExecutorTemplate executorTemplate = null;
    try {
        RowBatch rowBatch = param.getRowBatch();
        final Pipeline pipeline = getPipeline(rowBatch.getIdentity().getPipelineId());
        List<EventData> eventDatas = rowBatch.getDatas();
        // 使用set，提升remove时的查找速度
        final Set<EventData> removeDatas = Collections.synchronizedSet(new HashSet<EventData>());
        executorTemplate = executorTemplateGetter.get();
        executorTemplate.start();
        // 重新设置下poolSize
        executorTemplate.adjustPoolSize(pipeline.getParameters().getExtractPoolSize());
        for (final EventData eventData : eventDatas) {
            List<DataMediaPair> dataMediaPairs = ConfigHelper.findDataMediaPairByMediaId(pipeline, eventData.getTableId());
            if (dataMediaPairs == null) {
                throw new ExtractException("ERROR ## the dataMediaId = " + eventData.getTableId() + " dataMediaPair is null,please check");
            }
            for (DataMediaPair dataMediaPair : dataMediaPairs) {
                if (!dataMediaPair.isExistFilter()) {
                    continue;
                }
                final EventProcessor eventProcessor = extensionFactory.getExtension(EventProcessor.class, dataMediaPair.getFilterData());
                if (eventProcessor instanceof DataSourceFetcherAware) {
                    ((DataSourceFetcherAware) eventProcessor).setDataSourceFetcher(new DataSourceFetcher() {

                        @Override
                        public DataSource fetch(Long tableId) {
                            DataMedia dataMedia = ConfigHelper.findDataMedia(pipeline, tableId);
                            return dataSourceService.getDataSource(pipeline.getId(), dataMedia.getSource());
                        }
                    });
                    executorTemplate.submit(new Runnable() {

                        @Override
                        public void run() {
                            MDC.put(OtterConstants.splitPipelineLogFileKey, String.valueOf(pipeline.getId()));
                            boolean process = eventProcessor.process(eventData);
                            if (!process) {
                                // 添加到删除记录中
                                removeDatas.add(eventData);
                            }
                        }
                    });
                } else {
                    boolean process = eventProcessor.process(eventData);
                    if (!process) {
                        // 添加到删除记录中
                        removeDatas.add(eventData);
                        break;
                    }
                }
            }
        }
        // 等待所有都处理完成
        executorTemplate.waitForResult();
        if (!CollectionUtils.isEmpty(removeDatas)) {
            eventDatas.removeAll(removeDatas);
        }
    } finally {
        if (executorTemplate != null) {
            executorTemplateGetter.release(executorTemplate);
        }
    }
}