use of com.alibaba.otter.shared.etl.extend.processor.support.DataSourceFetcher in project otter by alibaba.
the class ProcessorExtractor method extract.
public void extract(DbBatch param) throws ExtractException {
ExecutorTemplate executorTemplate = null;
try {
RowBatch rowBatch = param.getRowBatch();
final Pipeline pipeline = getPipeline(rowBatch.getIdentity().getPipelineId());
List<EventData> eventDatas = rowBatch.getDatas();
// 使用set,提升remove时的查找速度
final Set<EventData> removeDatas = Collections.synchronizedSet(new HashSet<EventData>());
executorTemplate = executorTemplateGetter.get();
executorTemplate.start();
// 重新设置下poolSize
executorTemplate.adjustPoolSize(pipeline.getParameters().getExtractPoolSize());
for (final EventData eventData : eventDatas) {
List<DataMediaPair> dataMediaPairs = ConfigHelper.findDataMediaPairByMediaId(pipeline, eventData.getTableId());
if (dataMediaPairs == null) {
throw new ExtractException("ERROR ## the dataMediaId = " + eventData.getTableId() + " dataMediaPair is null,please check");
}
for (DataMediaPair dataMediaPair : dataMediaPairs) {
if (!dataMediaPair.isExistFilter()) {
continue;
}
final EventProcessor eventProcessor = extensionFactory.getExtension(EventProcessor.class, dataMediaPair.getFilterData());
if (eventProcessor instanceof DataSourceFetcherAware) {
((DataSourceFetcherAware) eventProcessor).setDataSourceFetcher(new DataSourceFetcher() {
@Override
public DataSource fetch(Long tableId) {
DataMedia dataMedia = ConfigHelper.findDataMedia(pipeline, tableId);
return dataSourceService.getDataSource(pipeline.getId(), dataMedia.getSource());
}
});
executorTemplate.submit(new Runnable() {
@Override
public void run() {
MDC.put(OtterConstants.splitPipelineLogFileKey, String.valueOf(pipeline.getId()));
boolean process = eventProcessor.process(eventData);
if (!process) {
// 添加到删除记录中
removeDatas.add(eventData);
}
}
});
} else {
boolean process = eventProcessor.process(eventData);
if (!process) {
// 添加到删除记录中
removeDatas.add(eventData);
break;
}
}
}
}
// 等待所有都处理完成
executorTemplate.waitForResult();
if (!CollectionUtils.isEmpty(removeDatas)) {
eventDatas.removeAll(removeDatas);
}
} finally {
if (executorTemplate != null) {
executorTemplateGetter.release(executorTemplate);
}
}
}
Aggregations