Examples with Pipeline com.alibaba.otter.shared.common.model.config.pipeline.Pipeline used on opensource projects

Example 91 with Pipeline

use of com.alibaba.otter.shared.common.model.config.pipeline.Pipeline in project otter by alibaba.

the class RestartAlarmRecovery method recovery.

public void recovery(AlarmRule alarmRule) {
    Pipeline pipeline = pipelineService.findById(alarmRule.getPipelineId());
    AlarmRecoveryDelayed delayed = new AlarmRecoveryDelayed(pipeline.getChannelId(), alarmRule.getId(), false, checkTime);
    // 做异步处理，避免并发时重复执行recovery
    synchronized (queue) {
        if (!queue.contains(delayed)) {
            queue.add(delayed);
        }
    }
}

Example 92 with Pipeline

use of com.alibaba.otter.shared.common.model.config.pipeline.Pipeline in project otter by alibaba.

the class ConfigRemoteServiceImpl method notifyChannel.

public boolean notifyChannel(final Channel channel) {
    Assert.notNull(channel);
    // 获取所有的Node节点
    NotifyChannelEvent event = new NotifyChannelEvent();
    event.setChannel(channel);
    Set<String> addrsSet = new HashSet<String>();
    // 组装当前pipeline下的存活的node节点
    for (Pipeline pipeline : channel.getPipelines()) {
        List<Node> nodes = new ArrayList<Node>();
        nodes.addAll(pipeline.getSelectNodes());
        nodes.addAll(pipeline.getExtractNodes());
        nodes.addAll(pipeline.getLoadNodes());
        for (Node node : nodes) {
            if (node.getStatus().isStart() && StringUtils.isNotEmpty(node.getIp()) && node.getPort() != 0) {
                String addr = node.getIp() + ":" + node.getPort();
                if (node.getParameters().getUseExternalIp()) {
                    addr = node.getParameters().getExternalIp() + ":" + node.getPort();
                }
                addrsSet.add(addr);
            }
        }
    }
    List<String> addrsList = new ArrayList<String>(addrsSet);
    if (CollectionUtils.isEmpty(addrsList) && channel.getStatus().isStart()) {
        throw new ManagerException("no live node for notifyChannel");
    } else if (CollectionUtils.isEmpty(addrsList)) {
        // 针对关闭操作，可直接处理
        return true;
    } else {
        // 做一下随机，避免每次选择的机器都是同一台
        Collections.shuffle(addrsList);
        try {
            String[] addrs = addrsList.toArray(new String[addrsList.size()]);
            // 推送配置
            List<Boolean> result = (List<Boolean>) communicationClient.call(addrs, event);
            logger.info("## notifyChannel to [{}] channel[{}] result[{}]", new Object[] { ArrayUtils.toString(addrs), channel.toString(), result });
            boolean flag = true;
            for (Boolean f : result) {
                flag &= f;
            }
            return flag;
        } catch (Exception e) {
            logger.error("## notifyChannel error!", e);
            throw new ManagerException(e);
        }
    }
}

Example 93 with Pipeline

use of com.alibaba.otter.shared.common.model.config.pipeline.Pipeline in project otter by alibaba.

the class OtterTransformerTest method test_rowData_oracle_mysql.

@Test
public void test_rowData_oracle_mysql() {
    final Pipeline pipeline = new Pipeline();
    pipeline.setId(100L);
    List<DataMediaPair> pairs = new ArrayList<DataMediaPair>();
    DataMediaPair pair1 = new DataMediaPair();
    pair1.setId(1L);
    pair1.setPipelineId(pipeline.getId());
    pair1.setPullWeight(1L);
    pair1.setPushWeight(1L);
    DbDataMedia oracleMedia = getOracleMedia();
    oracleMedia.setId(1L);
    pair1.setSource(oracleMedia);
    DbDataMedia mysqlMedia = getMysqlMedia();
    pair1.setTarget(mysqlMedia);
    pairs.add(pair1);
    pipeline.setPairs(pairs);
    PipelineParameter param = new PipelineParameter();
    param.setSyncMode(SyncMode.ROW);
    pipeline.setParameters(param);
    new NonStrictExpectations() {

        {
            configClientService.findPipeline(anyLong);
            returns(pipeline);
        }
    };
    Identity identity = new Identity();
    identity.setChannelId(100L);
    identity.setPipelineId(100L);
    identity.setProcessId(100L);
    RowBatch rowBatch = new RowBatch();
    rowBatch.setIdentity(identity);
    EventData eventData = new EventData();
    eventData.setTableId(1L);
    eventData.setSchemaName("srf");
    eventData.setTableName("columns");
    eventData.setEventType(EventType.UPDATE);
    eventData.setExecuteTime(100L);
    eventData.getKeys().add(buildColumn("id", Types.NUMERIC, "1", true, false));
    eventData.getKeys().add(buildColumn("name", Types.VARCHAR, "ljh", true, false));
    eventData.getColumns().add(buildColumn("alias_name", Types.CHAR, "hello", false, false));
    eventData.getColumns().add(buildColumn("amount", Types.NUMERIC, "100.01", false, false));
    eventData.getColumns().add(buildColumn("text_b", Types.BLOB, "[116,101,120,116,95,98]", false, false));
    eventData.getColumns().add(buildColumn("text_c", Types.CLOB, "text_c", false, false));
    eventData.getColumns().add(buildColumn("curr_date", Types.DATE, "2011-01-01", false, false));
    eventData.getColumns().add(buildColumn("gmt_create", Types.DATE, "2011-01-01 11:11:11", false, false));
    eventData.getColumns().add(buildColumn("gmt_modify", Types.DATE, "2011-01-01 11:11:11", false, false));
    rowBatch.merge(eventData);
    Map<Class, BatchObject> batchs = otterTransformFactory.transform(rowBatch);
    RowBatch result = (RowBatch) batchs.get(EventData.class);
    want.number(result.getDatas().size()).isEqualTo(1);
}

Example 94 with Pipeline

use of com.alibaba.otter.shared.common.model.config.pipeline.Pipeline in project otter by alibaba.

the class FileBatchConflictDetectServiceIntegration method test_localFile.

@Test
public void test_localFile() {
    final Pipeline pipeline = new Pipeline();
    pipeline.setId(100L);
    final Node currentNode = new Node();
    currentNode.setId(1L);
    new NonStrictExpectations() {

        {
            configClientService.currentNode();
            returns(currentNode);
            configClientService.findPipeline(anyLong);
            returns(pipeline);
        }
    };
    Identity identity = new Identity();
    identity.setChannelId(100L);
    identity.setPipelineId(100L);
    identity.setProcessId(100L);
    FileBatch fileBatch = new FileBatch();
    fileBatch.setIdentity(identity);
    fileBatch.getFiles().addAll(generatorLocalFileData("fileLoad", 10));
    FileBatch result = fileBatchConflictDetectService.detect(fileBatch, 1L);
    want.number(result.getFiles().size()).isEqualTo(0);
    NioUtils.delete(new File(tmp + File.separator + OTTERLOAD));
}

Example 95 with Pipeline

use of com.alibaba.otter.shared.common.model.config.pipeline.Pipeline in project otter by alibaba.

the class DbLoadActionTest method test_db_load_oracle.

@Test
public void test_db_load_oracle() {
    ArbitrateConfigRegistry.regist(configClientService);
    dbLoadAction = (DbLoadAction) TestedObject.getSpringBeanFactory().getBean("dbLoadAction");
    final Channel channel = new Channel();
    channel.setId(1L);
    final Pipeline pipeline = new Pipeline();
    pipeline.setId(100L);
    List<DataMediaPair> pairs = generatorDataMediaPairForOracle(20);
    pipeline.setPairs(pairs);
    pipeline.getParameters().merge(new SystemParameter());
    pipeline.getParameters().merge(new ChannelParameter());
    // final Pipeline oppositePipeline = new Pipeline();
    // oppositePipeline.setId(101L);
    channel.setPipelines(Arrays.asList(pipeline));
    final Node currentNode = new Node();
    currentNode.setId(1L);
    new NonStrictExpectations() {

        {
            configClientService.findChannel(anyLong);
            returns(channel);
            configClientService.findPipeline(anyLong);
            returns(pipeline);
            configClientService.currentNode();
            returns(currentNode);
        }
    };
    Identity identity = new Identity();
    identity.setChannelId(100L);
    identity.setPipelineId(100L);
    identity.setProcessId(100L);
    RowBatch rowBatch = new RowBatch();
    rowBatch.setIdentity(identity);
    List<EventData> eventDatas = generatorEventDataForOracle(0, 20, EventType.INSERT);
    for (EventData eventData : eventDatas) {
        rowBatch.merge(eventData);
    }
    eventDatas = generatorEventDataForOracle(10, 10, EventType.INSERT);
    for (EventData eventData : eventDatas) {
        rowBatch.merge(eventData);
    }
    eventDatas = generatorEventDataForOracle(19, 1, EventType.DELETE);
    for (EventData eventData : eventDatas) {
        rowBatch.merge(eventData);
    }
    WeightController controller = new WeightController(1);
    dbLoadAction.load(rowBatch, controller);
}