Examples with OuterAdapter com.alibaba.otter.canal.client.adapter.OuterAdapter used on opensource projects

Example 1 with OuterAdapter

use of com.alibaba.otter.canal.client.adapter.OuterAdapter in project canal by alibaba.

the class CanalAdapterLoader method loadAdapter.

private void loadAdapter(OuterAdapterConfig config, List<OuterAdapter> canalOutConnectors) {
    try {
        OuterAdapter adapter;
        adapter = loader.getExtension(config.getName(), StringUtils.trimToEmpty(config.getKey()));
        ClassLoader cl = Thread.currentThread().getContextClassLoader();
        // 替换ClassLoader
        Thread.currentThread().setContextClassLoader(adapter.getClass().getClassLoader());
        Environment env = (Environment) SpringContext.getBean(Environment.class);
        Properties evnProperties = null;
        if (env instanceof StandardEnvironment) {
            evnProperties = new Properties();
            for (PropertySource<?> propertySource : ((StandardEnvironment) env).getPropertySources()) {
                if (propertySource instanceof EnumerablePropertySource) {
                    String[] names = ((EnumerablePropertySource<?>) propertySource).getPropertyNames();
                    for (String name : names) {
                        Object val = env.getProperty(name);
                        if (val != null) {
                            evnProperties.put(name, val);
                        }
                    }
                }
            }
        }
        adapter.init(config, evnProperties);
        Thread.currentThread().setContextClassLoader(cl);
        canalOutConnectors.add(adapter);
        logger.info("Load canal adapter: {} succeed", config.getName());
    } catch (Exception e) {
        logger.error("Load canal adapter: {} failed", config.getName(), e);
    }
}

Example 2 with OuterAdapter

use of com.alibaba.otter.canal.client.adapter.OuterAdapter in project canal by alibaba.

the class CommonRest method etl.

/**
 * ETL curl http://127.0.0.1:8081/etl/rdb/oracle1/mytest_user.yml -X POST
 *
 * @param type 类型 hbase, es
 * @param key adapter key
 * @param task 任务名对应配置文件名 mytest_user.yml
 * @param params etl where条件参数, 为空全部导入
 */
@PostMapping("/etl/{type}/{key}/{task}")
public EtlResult etl(@PathVariable String type, @PathVariable String key, @PathVariable String task, @RequestParam(name = "params", required = false) String params) {
    OuterAdapter adapter = loader.getExtension(type, key);
    String destination = adapter.getDestination(task);
    String lockKey = destination == null ? task : destination;
    boolean locked = etlLock.tryLock(ETL_LOCK_ZK_NODE + type + "-" + lockKey);
    if (!locked) {
        EtlResult result = new EtlResult();
        result.setSucceeded(false);
        result.setErrorMessage(task + " 有其他进程正在导入中, 请稍后再试");
        return result;
    }
    try {
        boolean oriSwitchStatus;
        if (destination != null) {
            oriSwitchStatus = syncSwitch.status(destination);
            if (oriSwitchStatus) {
                syncSwitch.off(destination);
            }
        } else {
            // task可能为destination，直接锁task
            oriSwitchStatus = syncSwitch.status(task);
            if (oriSwitchStatus) {
                syncSwitch.off(task);
            }
        }
        try {
            List<String> paramArray = null;
            if (params != null) {
                paramArray = Arrays.asList(params.trim().split(";"));
            }
            return adapter.etl(task, paramArray);
        } finally {
            if (destination != null && oriSwitchStatus) {
                syncSwitch.on(destination);
            } else if (destination == null && oriSwitchStatus) {
                syncSwitch.on(task);
            }
        }
    } finally {
        etlLock.unlock(ETL_LOCK_ZK_NODE + type + "-" + lockKey);
    }
}

Example 3 with OuterAdapter

use of com.alibaba.otter.canal.client.adapter.OuterAdapter in project canal by alibaba.

the class CanalAdapterLoader method init.

/**
 * 初始化canal-client
 */
public void init() {
    loader = ExtensionLoader.getExtensionLoader(OuterAdapter.class);
    for (CanalClientConfig.CanalAdapter canalAdapter : canalClientConfig.getCanalAdapters()) {
        for (CanalClientConfig.Group group : canalAdapter.getGroups()) {
            List<List<OuterAdapter>> canalOuterAdapterGroups = new CopyOnWriteArrayList<>();
            List<OuterAdapter> canalOuterAdapters = new CopyOnWriteArrayList<>();
            for (OuterAdapterConfig config : group.getOuterAdapters()) {
                loadAdapter(config, canalOuterAdapters);
            }
            canalOuterAdapterGroups.add(canalOuterAdapters);
            AdapterProcessor adapterProcessor = canalAdapterProcessors.computeIfAbsent(canalAdapter.getInstance() + "|" + StringUtils.trimToEmpty(group.getGroupId()), f -> new AdapterProcessor(canalClientConfig, canalAdapter.getInstance(), group.getGroupId(), canalOuterAdapterGroups));
            adapterProcessor.start();
            logger.info("Start adapter for canal-client mq topic: {} succeed", canalAdapter.getInstance() + "-" + group.getGroupId());
        }
    }
// if ("tcp".equalsIgnoreCase(canalClientConfig.getMode())) {
// // 初始化canal-client的适配器
// for (CanalClientConfig.CanalAdapter canalAdapter :
// canalClientConfig.getCanalAdapters()) {
// List<List<OuterAdapter>> canalOuterAdapterGroups = new
// CopyOnWriteArrayList<>();
// 
// for (CanalClientConfig.Group connectorGroup :
// canalAdapter.getGroups()) {
// List<OuterAdapter> canalOutConnectors = new CopyOnWriteArrayList<>();
// for (OuterAdapterConfig c : connectorGroup.getOuterAdapters()) {
// loadAdapter(c, canalOutConnectors);
// }
// canalOuterAdapterGroups.add(canalOutConnectors);
// }
// CanalAdapterWorker worker;
// if (StringUtils.isNotEmpty(canalServerHost)) {
// worker = new CanalAdapterWorker(canalClientConfig,
// canalAdapter.getInstance(),
// canalServerHost,
// zkHosts,
// canalOuterAdapterGroups);
// } else if (zkHosts != null) {
// worker = new CanalAdapterWorker(canalClientConfig,
// canalAdapter.getInstance(),
// zkHosts,
// canalOuterAdapterGroups);
// } else {
// throw new RuntimeException("No canal server connector found");
// }
// canalWorkers.put(canalAdapter.getInstance(), worker);
// worker.start();
// logger.info("Start adapter for canal instance: {} succeed",
// canalAdapter.getInstance());
// }
// } else if ("kafka".equalsIgnoreCase(canalClientConfig.getMode())) {
// // 初始化canal-client-kafka的适配器
// for (CanalClientConfig.CanalAdapter canalAdapter :
// canalClientConfig.getCanalAdapters()) {
// for (CanalClientConfig.Group group : canalAdapter.getGroups()) {
// List<List<OuterAdapter>> canalOuterAdapterGroups = new
// CopyOnWriteArrayList<>();
// List<OuterAdapter> canalOuterAdapters = new CopyOnWriteArrayList<>();
// for (OuterAdapterConfig config : group.getOuterAdapters()) {
// loadAdapter(config, canalOuterAdapters);
// }
// canalOuterAdapterGroups.add(canalOuterAdapters);
// 
// CanalAdapterKafkaWorker canalKafkaWorker = new
// CanalAdapterKafkaWorker(canalClientConfig,
// canalClientConfig.getMqServers(),
// canalAdapter.getInstance(),
// group.getGroupId(),
// canalOuterAdapterGroups,
// canalClientConfig.getFlatMessage());
// canalMQWorker.put(canalAdapter.getInstance() + "-kafka-" +
// group.getGroupId(), canalKafkaWorker);
// canalKafkaWorker.start();
// logger.info("Start adapter for canal-client mq topic: {} succeed",
// canalAdapter.getInstance() + "-" + group.getGroupId());
// }
// }
// } else if ("rocketMQ".equalsIgnoreCase(canalClientConfig.getMode()))
// {
// // 初始化canal-client-rocketMQ的适配器
// for (CanalClientConfig.CanalAdapter canalAdapter :
// canalClientConfig.getCanalAdapters()) {
// for (CanalClientConfig.Group group : canalAdapter.getGroups()) {
// List<List<OuterAdapter>> canalOuterAdapterGroups = new
// CopyOnWriteArrayList<>();
// List<OuterAdapter> canalOuterAdapters = new CopyOnWriteArrayList<>();
// for (OuterAdapterConfig config : group.getOuterAdapters()) {
// loadAdapter(config, canalOuterAdapters);
// }
// canalOuterAdapterGroups.add(canalOuterAdapters);
// CanalAdapterRocketMQWorker rocketMQWorker = new
// CanalAdapterRocketMQWorker(canalClientConfig,
// canalClientConfig.getMqServers(),
// canalAdapter.getInstance(),
// group.getGroupId(),
// canalOuterAdapterGroups,
// canalClientConfig.getAccessKey(),
// canalClientConfig.getSecretKey(),
// canalClientConfig.getFlatMessage(),
// canalClientConfig.isEnableMessageTrace(),
// canalClientConfig.getCustomizedTraceTopic(),
// canalClientConfig.getAccessChannel(),
// canalClientConfig.getNamespace());
// canalMQWorker.put(canalAdapter.getInstance() + "-rocketmq-" +
// group.getGroupId(), rocketMQWorker);
// rocketMQWorker.start();
// 
// logger.info("Start adapter for canal-client mq topic: {} succeed",
// canalAdapter.getInstance() + "-" + group.getGroupId());
// }
// }
// } else if ("rabbitMQ".equalsIgnoreCase(canalClientConfig.getMode()))
// {
// // 初始化canal-client-rabbitMQ的适配器
// for (CanalClientConfig.CanalAdapter canalAdapter :
// canalClientConfig.getCanalAdapters()) {
// for (CanalClientConfig.Group group : canalAdapter.getGroups()) {
// List<List<OuterAdapter>> canalOuterAdapterGroups = new
// CopyOnWriteArrayList<>();
// List<OuterAdapter> canalOuterAdapters = new CopyOnWriteArrayList<>();
// for (OuterAdapterConfig config : group.getOuterAdapters()) {
// loadAdapter(config, canalOuterAdapters);
// }
// canalOuterAdapterGroups.add(canalOuterAdapters);
// CanalAdapterRabbitMQWorker rabbitMQWork = new
// CanalAdapterRabbitMQWorker(canalClientConfig,
// canalOuterAdapterGroups,
// canalAdapter.getInstance(),
// group.getGroupId(),
// canalClientConfig.getFlatMessage());
// canalMQWorker.put(canalAdapter.getInstance() + "-rabbitmq-" +
// group.getGroupId(), rabbitMQWork);
// rabbitMQWork.start();
// 
// logger.info("Start adapter for canal-client mq topic: {} succeed",
// canalAdapter.getInstance() + "-" + group.getGroupId());
// }
// }
// // CanalAdapterRabbitMQWork
// }
}