Search in sources :

Example 1 with DataSourceFetcher

use of com.alibaba.otter.shared.etl.extend.processor.support.DataSourceFetcher in project otter by alibaba.

the class ProcessorExtractor method extract.

public void extract(DbBatch param) throws ExtractException {
    ExecutorTemplate executorTemplate = null;
    try {
        RowBatch rowBatch = param.getRowBatch();
        final Pipeline pipeline = getPipeline(rowBatch.getIdentity().getPipelineId());
        List<EventData> eventDatas = rowBatch.getDatas();
        // 使用set,提升remove时的查找速度
        final Set<EventData> removeDatas = Collections.synchronizedSet(new HashSet<EventData>());
        executorTemplate = executorTemplateGetter.get();
        executorTemplate.start();
        // 重新设置下poolSize
        executorTemplate.adjustPoolSize(pipeline.getParameters().getExtractPoolSize());
        for (final EventData eventData : eventDatas) {
            List<DataMediaPair> dataMediaPairs = ConfigHelper.findDataMediaPairByMediaId(pipeline, eventData.getTableId());
            if (dataMediaPairs == null) {
                throw new ExtractException("ERROR ## the dataMediaId = " + eventData.getTableId() + " dataMediaPair is null,please check");
            }
            for (DataMediaPair dataMediaPair : dataMediaPairs) {
                if (!dataMediaPair.isExistFilter()) {
                    continue;
                }
                final EventProcessor eventProcessor = extensionFactory.getExtension(EventProcessor.class, dataMediaPair.getFilterData());
                if (eventProcessor instanceof DataSourceFetcherAware) {
                    ((DataSourceFetcherAware) eventProcessor).setDataSourceFetcher(new DataSourceFetcher() {

                        @Override
                        public DataSource fetch(Long tableId) {
                            DataMedia dataMedia = ConfigHelper.findDataMedia(pipeline, tableId);
                            return dataSourceService.getDataSource(pipeline.getId(), dataMedia.getSource());
                        }
                    });
                    executorTemplate.submit(new Runnable() {

                        @Override
                        public void run() {
                            MDC.put(OtterConstants.splitPipelineLogFileKey, String.valueOf(pipeline.getId()));
                            boolean process = eventProcessor.process(eventData);
                            if (!process) {
                                // 添加到删除记录中
                                removeDatas.add(eventData);
                            }
                        }
                    });
                } else {
                    boolean process = eventProcessor.process(eventData);
                    if (!process) {
                        // 添加到删除记录中
                        removeDatas.add(eventData);
                        break;
                    }
                }
            }
        }
        // 等待所有都处理完成
        executorTemplate.waitForResult();
        if (!CollectionUtils.isEmpty(removeDatas)) {
            eventDatas.removeAll(removeDatas);
        }
    } finally {
        if (executorTemplate != null) {
            executorTemplateGetter.release(executorTemplate);
        }
    }
}
Also used : ExtractException(com.alibaba.otter.node.etl.extract.exceptions.ExtractException) ExecutorTemplate(com.alibaba.otter.shared.common.utils.thread.ExecutorTemplate) DataMediaPair(com.alibaba.otter.shared.common.model.config.data.DataMediaPair) DataSourceFetcher(com.alibaba.otter.shared.etl.extend.processor.support.DataSourceFetcher) DataSourceFetcherAware(com.alibaba.otter.shared.etl.extend.processor.support.DataSourceFetcherAware) EventData(com.alibaba.otter.shared.etl.model.EventData) Pipeline(com.alibaba.otter.shared.common.model.config.pipeline.Pipeline) DataSource(javax.sql.DataSource) RowBatch(com.alibaba.otter.shared.etl.model.RowBatch) EventProcessor(com.alibaba.otter.shared.etl.extend.processor.EventProcessor) DataMedia(com.alibaba.otter.shared.common.model.config.data.DataMedia)

Aggregations

ExtractException (com.alibaba.otter.node.etl.extract.exceptions.ExtractException)1 DataMedia (com.alibaba.otter.shared.common.model.config.data.DataMedia)1 DataMediaPair (com.alibaba.otter.shared.common.model.config.data.DataMediaPair)1 Pipeline (com.alibaba.otter.shared.common.model.config.pipeline.Pipeline)1 ExecutorTemplate (com.alibaba.otter.shared.common.utils.thread.ExecutorTemplate)1 EventProcessor (com.alibaba.otter.shared.etl.extend.processor.EventProcessor)1 DataSourceFetcher (com.alibaba.otter.shared.etl.extend.processor.support.DataSourceFetcher)1 DataSourceFetcherAware (com.alibaba.otter.shared.etl.extend.processor.support.DataSourceFetcherAware)1 EventData (com.alibaba.otter.shared.etl.model.EventData)1 RowBatch (com.alibaba.otter.shared.etl.model.RowBatch)1 DataSource (javax.sql.DataSource)1