use of com.alibaba.otter.node.etl.load.loader.weight.WeightController in project otter by alibaba.
the class FileLoadActionTest method testLoadWithLocal.
@Test
public void testLoadWithLocal() throws IOException {
// 构造fileData使用的参数,fileDataStartIndex 决定着 pipeline 与 fileData 对应的关系(通过
// dataMediaPair 的 id),
// 以及 dataMediaPair 的 pushWeight
final int fileDataStartIndex = 0;
final int fileDataCount = 50;
final Pipeline pipeline = buildPipeline(fileDataStartIndex, fileDataCount);
final Channel channel = new Channel();
new NonStrictExpectations() {
{
configClientService.findChannel(anyLong);
returns(channel);
configClientService.findPipeline(anyLong);
returns(pipeline);
}
};
Identity id = buildIdentity(1L, 2L, 3L);
FileBatch fileBatch = buildFileBatch(id);
fileBatch.getFiles().addAll(buildFileDatas(null, EventType.INSERT, fileDataStartIndex, fileDataCount, true));
WeightController controller = new WeightController(1);
FileLoadContext context = fileLoadAction.load(fileBatch, ROOT_DIR, controller);
want.object(context.getChannel()).isEqualTo(channel);
want.object(context.getPipeline()).isEqualTo(pipeline);
want.object(context.getPrepareDatas()).isEqualTo(fileBatch.getFiles());
want.number(context.getProcessedDatas().size()).isEqualTo(fileBatch.getFiles().size());
}
use of com.alibaba.otter.node.etl.load.loader.weight.WeightController in project otter by alibaba.
the class DbLoadActionTest method test_db_load_mysql.
@Test
public void test_db_load_mysql() {
ArbitrateConfigRegistry.regist(configClientService);
dbLoadAction = (DbLoadAction) TestedObject.getSpringBeanFactory().getBean("dbLoadAction");
final Channel channel = new Channel();
channel.setId(1L);
final Pipeline pipeline = new Pipeline();
pipeline.setId(100L);
List<DataMediaPair> pairs = generatorDataMediaPairForMysql(20);
pipeline.setPairs(pairs);
pipeline.getParameters().merge(new SystemParameter());
pipeline.getParameters().merge(new ChannelParameter());
// pipeline.getParameters().setChannelInfo("LJH_DEMO");
// final Pipeline oppositePipeline = new Pipeline();
// oppositePipeline.setId(101L);
channel.setPipelines(Arrays.asList(pipeline));
final Node currentNode = new Node();
currentNode.setId(1L);
new NonStrictExpectations() {
{
configClientService.findChannel(anyLong);
returns(channel);
configClientService.findPipeline(anyLong);
returns(pipeline);
configClientService.currentNode();
returns(currentNode);
}
};
Identity identity = new Identity();
identity.setChannelId(100L);
identity.setPipelineId(100L);
identity.setProcessId(100L);
RowBatch rowBatch = new RowBatch();
rowBatch.setIdentity(identity);
List<EventData> eventDatas = generatorEventDataForMysql(0, 20, EventType.INSERT);
for (EventData eventData : eventDatas) {
rowBatch.merge(eventData);
}
eventDatas = generatorEventDataForMysql(10, 10, EventType.INSERT);
for (EventData eventData : eventDatas) {
rowBatch.merge(eventData);
}
eventDatas = generatorEventDataForMysql(19, 1, EventType.DELETE);
for (EventData eventData : eventDatas) {
rowBatch.merge(eventData);
}
WeightController controller = new WeightController(1);
dbLoadAction.load(rowBatch, controller);
}
use of com.alibaba.otter.node.etl.load.loader.weight.WeightController in project otter by alibaba.
the class DataBatchLoader method load.
public List<LoadContext> load(DbBatch data) {
final RowBatch rowBatch = data.getRowBatch();
final FileBatch fileBatch = data.getFileBatch();
boolean existFileBatch = (rowBatch != null && !CollectionUtils.isEmpty(fileBatch.getFiles()) && data.getRoot() != null);
boolean existRowBatch = (rowBatch != null && !CollectionUtils.isEmpty(rowBatch.getDatas()));
int count = 0;
List<RowBatch> rowBatchs = null;
if (existRowBatch) {
// 根据介质内容进行分类合并,每个介质一个载入通道
rowBatchs = split(rowBatch);
count += rowBatchs.size();
}
if (existFileBatch) {
count += 1;
}
WeightController controller = new WeightController(count);
List<Future> futures = new ArrayList<Future>();
ExecutorCompletionService completionService = new ExecutorCompletionService(executorService);
if (existFileBatch) {
submitFileBatch(futures, completionService, fileBatch, data.getRoot(), controller);
}
if (existRowBatch) {
submitRowBatch(futures, completionService, rowBatchs, controller);
}
// 先获取一下异步处理的结果,记录一下出错的index
List<LoadContext> processedContexts = new ArrayList<LoadContext>();
int index = 0;
LoadException exception = null;
while (index < futures.size()) {
try {
// 它也可能被打断
Future future = completionService.take();
future.get();
} catch (InterruptedException e) {
exception = new LoadException(e);
break;
} catch (ExecutionException e) {
exception = new LoadException(e);
break;
}
index++;
}
// 任何一个线程返回,出现了异常,就退出整个调度
if (index < futures.size()) {
// 小于代表有错误,需要对未完成的记录进行cancel操作,对已完成的结果进行收集,做重复录入过滤记录
for (int errorIndex = 0; errorIndex < futures.size(); errorIndex++) {
Future future = futures.get(errorIndex);
if (future.isDone()) {
try {
LoadContext loadContext = (LoadContext) future.get();
if (loadContext instanceof DbLoadContext) {
// 做一下出错处理,记录到store中
dbInterceptor.error((DbLoadContext) loadContext);
}
} catch (InterruptedException e) {
// ignore
} catch (ExecutionException e) {
// ignore
} catch (Exception e) {
logger.error("interceptor process error failed", e);
}
} else {
// 对未完成的进行取消
future.cancel(true);
}
}
} else {
for (int i = 0; i < futures.size(); i++) {
// 收集一下正确处理完成的结果
Future future = futures.get(i);
try {
LoadContext loadContext = (LoadContext) future.get();
if (loadContext instanceof DbLoadContext) {
processedContexts.add((DbLoadContext) loadContext);
}
} catch (InterruptedException e) {
// ignore
} catch (ExecutionException e) {
// ignore
}
}
}
if (exception != null) {
throw exception;
} else {
return processedContexts;
}
}
use of com.alibaba.otter.node.etl.load.loader.weight.WeightController in project otter by alibaba.
the class DbLoadActionTest method test_db_load_oracle.
@Test
public void test_db_load_oracle() {
ArbitrateConfigRegistry.regist(configClientService);
dbLoadAction = (DbLoadAction) TestedObject.getSpringBeanFactory().getBean("dbLoadAction");
final Channel channel = new Channel();
channel.setId(1L);
final Pipeline pipeline = new Pipeline();
pipeline.setId(100L);
List<DataMediaPair> pairs = generatorDataMediaPairForOracle(20);
pipeline.setPairs(pairs);
pipeline.getParameters().merge(new SystemParameter());
pipeline.getParameters().merge(new ChannelParameter());
// final Pipeline oppositePipeline = new Pipeline();
// oppositePipeline.setId(101L);
channel.setPipelines(Arrays.asList(pipeline));
final Node currentNode = new Node();
currentNode.setId(1L);
new NonStrictExpectations() {
{
configClientService.findChannel(anyLong);
returns(channel);
configClientService.findPipeline(anyLong);
returns(pipeline);
configClientService.currentNode();
returns(currentNode);
}
};
Identity identity = new Identity();
identity.setChannelId(100L);
identity.setPipelineId(100L);
identity.setProcessId(100L);
RowBatch rowBatch = new RowBatch();
rowBatch.setIdentity(identity);
List<EventData> eventDatas = generatorEventDataForOracle(0, 20, EventType.INSERT);
for (EventData eventData : eventDatas) {
rowBatch.merge(eventData);
}
eventDatas = generatorEventDataForOracle(10, 10, EventType.INSERT);
for (EventData eventData : eventDatas) {
rowBatch.merge(eventData);
}
eventDatas = generatorEventDataForOracle(19, 1, EventType.DELETE);
for (EventData eventData : eventDatas) {
rowBatch.merge(eventData);
}
WeightController controller = new WeightController(1);
dbLoadAction.load(rowBatch, controller);
}
use of com.alibaba.otter.node.etl.load.loader.weight.WeightController in project otter by alibaba.
the class WeightWorkerTest method test_simple.
@Test
public void test_simple() {
int thread = 10;
int count = 10;
WeightController controller = new WeightController(thread);
CountDownLatch latch = new CountDownLatch(thread);
WeightWorkerTest[] workers = new WeightWorkerTest[thread];
for (int i = 0; i < thread; i++) {
int[] weights = new int[count];
for (int j = 0; j < count; j++) {
weights[j] = RandomUtils.nextInt(count);
}
workers[i] = new WeightWorkerTest(i, weights, controller, latch);
}
for (int i = 0; i < thread; i++) {
workers[i].start();
}
try {
latch.await();
} catch (InterruptedException e) {
want.fail();
}
}
Aggregations