Examples with CanalMetaManagerException com.alibaba.otter.canal.meta.exception.CanalMetaManagerException used on opensource projects

Example 1 with CanalMetaManagerException

use of com.alibaba.otter.canal.meta.exception.CanalMetaManagerException in project canal by alibaba.

the class FileMixedMetaManager method flushDataToFile.

private void flushDataToFile(String destination, File dataFile) {
    FileMetaInstanceData data = new FileMetaInstanceData();
    if (destinations.containsKey(destination)) {
        synchronized (destination.intern()) {
            // 基于destination控制一下并发更新
            data.setDestination(destination);
            List<FileMetaClientIdentityData> clientDatas = Lists.newArrayList();
            List<ClientIdentity> clientIdentitys = destinations.get(destination);
            for (ClientIdentity clientIdentity : clientIdentitys) {
                FileMetaClientIdentityData clientData = new FileMetaClientIdentityData();
                clientData.setClientIdentity(clientIdentity);
                Position position = cursors.get(clientIdentity);
                if (position != null && position != nullCursor) {
                    clientData.setCursor((LogPosition) position);
                }
                clientDatas.add(clientData);
            }
            data.setClientDatas(clientDatas);
        }
        String json = JsonUtils.marshalToString(data);
        try {
            FileUtils.writeStringToFile(dataFile, json);
        } catch (IOException e) {
            throw new CanalMetaManagerException(e);
        }
    }
}

Example 2 with CanalMetaManagerException

use of com.alibaba.otter.canal.meta.exception.CanalMetaManagerException in project canal by alibaba.

the class FileMixedMetaManager method start.

public void start() {
    super.start();
    Assert.notNull(dataDir);
    if (!dataDir.exists()) {
        try {
            FileUtils.forceMkdir(dataDir);
        } catch (IOException e) {
            throw new CanalMetaManagerException(e);
        }
    }
    if (!dataDir.canRead() || !dataDir.canWrite()) {
        throw new CanalMetaManagerException("dir[" + dataDir.getPath() + "] can not read/write");
    }
    dataFileCaches = MigrateMap.makeComputingMap(new Function<String, File>() {

        public File apply(String destination) {
            return getDataFile(destination);
        }
    });
    executor = Executors.newScheduledThreadPool(1);
    destinations = MigrateMap.makeComputingMap(new Function<String, List<ClientIdentity>>() {

        public List<ClientIdentity> apply(String destination) {
            return loadClientIdentity(destination);
        }
    });
    cursors = MigrateMap.makeComputingMap(new Function<ClientIdentity, Position>() {

        public Position apply(ClientIdentity clientIdentity) {
            Position position = loadCursor(clientIdentity.getDestination(), clientIdentity);
            if (position == null) {
                // 返回一个空对象标识，避免出现异常
                return nullCursor;
            } else {
                return position;
            }
        }
    });
    updateCursorTasks = Collections.synchronizedSet(new HashSet<ClientIdentity>());
    // 启动定时工作任务
    executor.scheduleAtFixedRate(new Runnable() {

        public void run() {
            List<ClientIdentity> tasks = new ArrayList<ClientIdentity>(updateCursorTasks);
            for (ClientIdentity clientIdentity : tasks) {
                MDC.put("destination", String.valueOf(clientIdentity.getDestination()));
                try {
                    // 定时将内存中的最新值刷到file中，多次变更只刷一次
                    if (logger.isInfoEnabled()) {
                        LogPosition cursor = (LogPosition) getCursor(clientIdentity);
                        logger.info("clientId:{} cursor:[{},{},{}] address[{}]", new Object[] { clientIdentity.getClientId(), cursor.getPostion().getJournalName(), cursor.getPostion().getPosition(), cursor.getPostion().getTimestamp(), cursor.getIdentity().getSourceAddress().toString() });
                    }
                    flushDataToFile(clientIdentity.getDestination());
                    updateCursorTasks.remove(clientIdentity);
                } catch (Throwable e) {
                    // ignore
                    logger.error("period update" + clientIdentity.toString() + " curosr failed!", e);
                }
            }
        }
    }, period, period, TimeUnit.MILLISECONDS);
}

Example 3 with CanalMetaManagerException

use of com.alibaba.otter.canal.meta.exception.CanalMetaManagerException in project canal by alibaba.

the class ZooKeeperMetaManager method subscribe.

public void subscribe(ClientIdentity clientIdentity) throws CanalMetaManagerException {
    String path = ZookeeperPathUtils.getClientIdNodePath(clientIdentity.getDestination(), clientIdentity.getClientId());
    try {
        zkClientx.createPersistent(path, true);
    } catch (ZkNodeExistsException e) {
    // ignore
    }
    if (clientIdentity.hasFilter()) {
        String filterPath = ZookeeperPathUtils.getFilterPath(clientIdentity.getDestination(), clientIdentity.getClientId());
        byte[] bytes = null;
        try {
            bytes = clientIdentity.getFilter().getBytes(ENCODE);
        } catch (UnsupportedEncodingException e) {
            throw new CanalMetaManagerException(e);
        }
        try {
            zkClientx.createPersistent(filterPath, bytes);
        } catch (ZkNodeExistsException e) {
            // ignore
            zkClientx.writeData(filterPath, bytes);
        }
    }
}

Example 4 with CanalMetaManagerException

use of com.alibaba.otter.canal.meta.exception.CanalMetaManagerException in project canal by alibaba.

the class ZooKeeperMetaManager method listAllSubscribeInfo.

public List<ClientIdentity> listAllSubscribeInfo(String destination) throws CanalMetaManagerException {
    String path = ZookeeperPathUtils.getDestinationPath(destination);
    List<String> childs = null;
    try {
        childs = zkClientx.getChildren(path);
    } catch (ZkNoNodeException e) {
    // ignore
    }
    if (CollectionUtils.isEmpty(childs)) {
        return new ArrayList<ClientIdentity>();
    }
    List<Short> clientIds = new ArrayList<Short>();
    for (String child : childs) {
        if (StringUtils.isNumeric(child)) {
            clientIds.add(ZookeeperPathUtils.getClientId(child));
        }
    }
    // 进行一个排序
    Collections.sort(clientIds);
    List<ClientIdentity> clientIdentities = Lists.newArrayList();
    for (Short clientId : clientIds) {
        path = ZookeeperPathUtils.getFilterPath(destination, clientId);
        byte[] bytes = zkClientx.readData(path, true);
        String filter = null;
        if (bytes != null) {
            try {
                filter = new String(bytes, ENCODE);
            } catch (UnsupportedEncodingException e) {
                throw new CanalMetaManagerException(e);
            }
        }
        clientIdentities.add(new ClientIdentity(destination, clientId, filter));
    }
    return clientIdentities;
}

Example 5 with CanalMetaManagerException

use of com.alibaba.otter.canal.meta.exception.CanalMetaManagerException in project canal by alibaba.

the class ZooKeeperMetaManager method removeBatch.

public PositionRange removeBatch(ClientIdentity clientIdentity, Long batchId) throws CanalMetaManagerException {
    String batchsPath = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(), clientIdentity.getClientId());
    List<String> nodes = zkClientx.getChildren(batchsPath);
    if (CollectionUtils.isEmpty(nodes)) {
        // 没有batch记录
        return null;
    }
    // 找到最小的Id
    ArrayList<Long> batchIds = new ArrayList<Long>(nodes.size());
    for (String batchIdString : nodes) {
        batchIds.add(Long.valueOf(batchIdString));
    }
    Long minBatchId = Collections.min(batchIds);
    if (!minBatchId.equals(batchId)) {
        // 检查一下提交的ack/rollback，必须按batchId分出去的顺序提交，否则容易出现丢数据
        throw new CanalMetaManagerException(String.format("batchId:%d is not the firstly:%d", batchId, minBatchId));
    }
    if (!batchIds.contains(batchId)) {
        // 不存在对应的batchId
        return null;
    }
    PositionRange positionRange = getBatch(clientIdentity, batchId);
    if (positionRange != null) {
        String path = ZookeeperPathUtils.getBatchMarkWithIdPath(clientIdentity.getDestination(), clientIdentity.getClientId(), batchId);
        zkClientx.delete(path);
    }
    return positionRange;
}