Examples with MappingConfig com.alibaba.otter.canal.client.adapter.hbase.config.MappingConfig used on opensource projects

Example 1 with MappingConfig

use of com.alibaba.otter.canal.client.adapter.hbase.config.MappingConfig in project canal by alibaba.

the class HbaseAdapter method count.

@Override
public Map<String, Object> count(String task) {
    MappingConfig config = hbaseMapping.get(task);
    String hbaseTable = config.getHbaseMapping().getHbaseTable();
    long rowCount = 0L;
    try {
        HTable table = (HTable) hbaseTemplate.getConnection().getTable(TableName.valueOf(hbaseTable));
        Scan scan = new Scan();
        scan.setFilter(new FirstKeyOnlyFilter());
        ResultScanner resultScanner = table.getScanner(scan);
        for (Result result : resultScanner) {
            rowCount += result.size();
        }
    } catch (IOException e) {
        logger.error(e.getMessage(), e);
    }
    Map<String, Object> res = new LinkedHashMap<>();
    res.put("hbaseTable", hbaseTable);
    res.put("count", rowCount);
    return res;
}

Example 2 with MappingConfig

use of com.alibaba.otter.canal.client.adapter.hbase.config.MappingConfig in project canal by alibaba.

the class HbaseAdapter method init.

@Override
public void init(OuterAdapterConfig configuration, Properties envProperties) {
    try {
        this.envProperties = envProperties;
        Map<String, MappingConfig> hbaseMappingTmp = MappingConfigLoader.load(envProperties);
        // 过滤不匹配的key的配置
        hbaseMappingTmp.forEach((key, mappingConfig) -> {
            if ((mappingConfig.getOuterAdapterKey() == null && configuration.getKey() == null) || (mappingConfig.getOuterAdapterKey() != null && mappingConfig.getOuterAdapterKey().equalsIgnoreCase(configuration.getKey()))) {
                hbaseMapping.put(key, mappingConfig);
            }
        });
        for (Map.Entry<String, MappingConfig> entry : hbaseMapping.entrySet()) {
            String configName = entry.getKey();
            MappingConfig mappingConfig = entry.getValue();
            String k;
            if (envProperties != null && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) {
                k = StringUtils.trimToEmpty(mappingConfig.getDestination()) + "-" + StringUtils.trimToEmpty(mappingConfig.getGroupId()) + "_" + mappingConfig.getHbaseMapping().getDatabase() + "-" + mappingConfig.getHbaseMapping().getTable();
            } else {
                k = StringUtils.trimToEmpty(mappingConfig.getDestination()) + "_" + mappingConfig.getHbaseMapping().getDatabase() + "-" + mappingConfig.getHbaseMapping().getTable();
            }
            Map<String, MappingConfig> configMap = mappingConfigCache.computeIfAbsent(k, k1 -> new ConcurrentHashMap<>());
            configMap.put(configName, mappingConfig);
        }
        Map<String, String> properties = configuration.getProperties();
        Configuration hbaseConfig = HBaseConfiguration.create();
        properties.forEach(hbaseConfig::set);
        hbaseTemplate = new HbaseTemplate(hbaseConfig);
        hbaseSyncService = new HbaseSyncService(hbaseTemplate);
        configMonitor = new HbaseConfigMonitor();
        configMonitor.init(this, envProperties);
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}

Example 3 with MappingConfig

use of com.alibaba.otter.canal.client.adapter.hbase.config.MappingConfig in project canal by alibaba.

the class HbaseAdapter method etl.

@Override
public EtlResult etl(String task, List<String> params) {
    EtlResult etlResult = new EtlResult();
    MappingConfig config = hbaseMapping.get(task);
    HbaseEtlService hbaseEtlService = new HbaseEtlService(hbaseTemplate, config);
    if (config != null) {
        return hbaseEtlService.importData(params);
    } else {
        StringBuilder resultMsg = new StringBuilder();
        boolean resSucc = true;
        for (MappingConfig configTmp : hbaseMapping.values()) {
            // 取所有的destination为task的配置
            if (configTmp.getDestination().equals(task)) {
                EtlResult etlRes = hbaseEtlService.importData(params);
                if (!etlRes.getSucceeded()) {
                    resSucc = false;
                    resultMsg.append(etlRes.getErrorMessage()).append("\n");
                } else {
                    resultMsg.append(etlRes.getResultMessage()).append("\n");
                }
            }
        }
        if (resultMsg.length() > 0) {
            etlResult.setSucceeded(resSucc);
            if (resSucc) {
                etlResult.setResultMessage(resultMsg.toString());
            } else {
                etlResult.setErrorMessage(resultMsg.toString());
            }
            return etlResult;
        }
    }
    etlResult.setSucceeded(false);
    etlResult.setErrorMessage("Task not found");
    return etlResult;
}