Examples with Pipeline com.alibaba.otter.shared.common.model.config.pipeline.Pipeline used on opensource projects

Example 51 with Pipeline

use of com.alibaba.otter.shared.common.model.config.pipeline.Pipeline in project otter by alibaba.

the class FreedomExtractorTest method test_mysql.

@Test
public void test_mysql() {
    final Pipeline pipeline = new Pipeline();
    pipeline.setId(100L);
    int start = RandomUtils.nextInt();
    int count = 10;
    List<DataMediaPair> pairs = getDataMediaPairForMysql(start, count);
    pipeline.setPairs(pairs);
    new NonStrictExpectations() {

        {
            configClientService.findPipeline(100L);
            returns(pipeline);
        }
    };
    // 构造数据
    RowBatch rowBatch = new RowBatch();
    rowBatch.setIdentity(identity);
    for (int tableId = start; tableId < start + count; tableId++) {
        for (int i = start; i < start + count; i++) {
            EventData eventData = getEventData(tableId, i);
            eventData.setSchemaName("retl");
            eventData.setTableName("retl_buffer");
            rowBatch.merge(eventData);
        }
    }
    DbBatch dbBatch = new DbBatch(rowBatch);
    freedomExtractor.extract(dbBatch);
    want.collection(dbBatch.getRowBatch().getDatas()).sizeEq(count * count);
}

Example 52 with Pipeline

use of com.alibaba.otter.shared.common.model.config.pipeline.Pipeline in project otter by alibaba.

the class ChannelArbitrateEvent method termin.

// ===================== help method =================
/**
 * 执行结束同步任务操作
 */
private Boolean termin(Long channelId, final TerminType type) throws Exception {
    Channel channel = ArbitrateConfigUtils.getChannelByChannelId(channelId);
    List<Pipeline> pipelines = channel.getPipelines();
    List<Future<Boolean>> futures = new ArrayList<Future<Boolean>>();
    for (final Pipeline pipeline : pipelines) {
        futures.add(arbitrateExecutor.submit(new Callable<Boolean>() {

            public Boolean call() {
                TerminEventData data = new TerminEventData();
                data.setPipelineId(pipeline.getId());
                data.setType(type);
                data.setCode("channel");
                data.setDesc(type.toString());
                // 处理关闭
                return errorTerminProcess.process(data);
            }
        }));
    }
    boolean result = false;
    Exception exception = null;
    int index = 0;
    for (Future<Boolean> future : futures) {
        try {
            // 进行处理
            result |= future.get();
        } catch (InterruptedException e) {
            // ignore
            Thread.currentThread().interrupt();
        } catch (ExecutionException e) {
            sendWarningMessage(pipelines.get(index).getId(), e);
            exception = e;
        }
        index++;
    }
    if (exception != null) {
        throw exception;
    } else {
        return result;
    }
}

Example 53 with Pipeline

use of com.alibaba.otter.shared.common.model.config.pipeline.Pipeline in project otter by alibaba.

the class AbstractLoadBalance method getExtractAliveNodes.

public List<Node> getExtractAliveNodes() {
    Pipeline pipeline = ArbitrateConfigUtils.getPipeline(getPipelineId());
    List<Node> extractNodes = pipeline.getExtractNodes();
    List<Node> eNodes = new ArrayList<Node>();
    List<Long> aliveNodes = nodeMonitor.getAliveNodes();
    for (Node sourceNode : extractNodes) {
        if (aliveNodes.contains(sourceNode.getId())) {
            eNodes.add(sourceNode);
        }
    }
    return eNodes;
}

Example 54 with Pipeline

use of com.alibaba.otter.shared.common.model.config.pipeline.Pipeline in project otter by alibaba.

the class AbstractLoadBalance method getTransformAliveNodes.

public List<Node> getTransformAliveNodes() {
    Pipeline pipeline = ArbitrateConfigUtils.getPipeline(getPipelineId());
    List<Node> transformNodes = pipeline.getLoadNodes();
    List<Node> tNodes = new ArrayList<Node>();
    List<Long> aliveNodes = nodeMonitor.getAliveNodes();
    for (Node sourceNode : transformNodes) {
        if (aliveNodes.contains(sourceNode.getId())) {
            tNodes.add(sourceNode);
        }
    }
    return tNodes;
}

Example 55 with Pipeline

use of com.alibaba.otter.shared.common.model.config.pipeline.Pipeline in project otter by alibaba.

the class OtterTransformerFactory method transform.

/**
 * 将一种源数据进行转化，最后得到的结果会根据DataMediaPair中定义的目标对象生成不同的数据对象 <br/>
 *
 * <pre>
 * 返回对象格式：Map
 * key : Class对象，代表生成的目标数据对象
 * value : 每种目标数据对象的集合数据
 * </pre>
 */
public Map<Class, BatchObject> transform(RowBatch rowBatch) {
    final Identity identity = translateIdentity(rowBatch.getIdentity());
    Map<Class, BatchObject> result = new HashMap<Class, BatchObject>();
    // 初始化默认值
    result.put(EventData.class, initBatchObject(identity, EventData.class));
    for (EventData eventData : rowBatch.getDatas()) {
        // 处理eventData
        Long tableId = eventData.getTableId();
        Pipeline pipeline = configClientService.findPipeline(identity.getPipelineId());
        // 针对每个同步数据，可能会存在多路复制的情况
        List<DataMediaPair> dataMediaPairs = ConfigHelper.findDataMediaPairByMediaId(pipeline, tableId);
        for (DataMediaPair pair : dataMediaPairs) {
            if (!pair.getSource().getId().equals(tableId)) {
                // 过滤tableID不为源的同步
                continue;
            }
            OtterTransformer translate = lookup(pair.getSource(), pair.getTarget());
            // 进行转化
            Object item = translate.transform(eventData, new OtterTransformerContext(identity, pair, pipeline));
            if (item == null) {
                continue;
            }
            // 合并结果
            merge(identity, result, item);
        }
    }
    return result;
}