Examples with ClientIdentity com.alibaba.otter.canal.protocol.ClientIdentity used on opensource projects

Example 1 with ClientIdentity

use of com.alibaba.otter.canal.protocol.ClientIdentity in project canal by alibaba.

the class AbstractMetaManagerTest method doSubscribeTest.

public void doSubscribeTest(CanalMetaManager metaManager) {
    ClientIdentity client1 = new ClientIdentity(destination, (short) 1);
    metaManager.subscribe(client1);
    // 重复调用
    metaManager.subscribe(client1);
    ClientIdentity client2 = new ClientIdentity(destination, (short) 2);
    metaManager.subscribe(client2);
    List<ClientIdentity> clients = metaManager.listAllSubscribeInfo(destination);
    Assert.assertEquals(Arrays.asList(client1, client2), clients);
    metaManager.unsubscribe(client2);
    ClientIdentity client3 = new ClientIdentity(destination, (short) 3);
    metaManager.subscribe(client3);
    clients = metaManager.listAllSubscribeInfo(destination);
    Assert.assertEquals(Arrays.asList(client1, client3), clients);
}

Example 2 with ClientIdentity

use of com.alibaba.otter.canal.protocol.ClientIdentity in project canal by alibaba.

the class MetaLogPositionManager method getLatestIndexBy.

public LogPosition getLatestIndexBy(String destination) {
    List<ClientIdentity> clientIdentitys = metaManager.listAllSubscribeInfo(destination);
    LogPosition result = null;
    if (!CollectionUtils.isEmpty(clientIdentitys)) {
        // 尝试找到一个最小的logPosition
        for (ClientIdentity clientIdentity : clientIdentitys) {
            LogPosition position = (LogPosition) metaManager.getCursor(clientIdentity);
            if (position == null) {
                continue;
            }
            if (result == null) {
                result = position;
            } else {
                result = CanalEventUtils.min(result, position);
            }
        }
    }
    return result;
}

Example 3 with ClientIdentity

use of com.alibaba.otter.canal.protocol.ClientIdentity in project canal by alibaba.

the class FileMixedMetaManager method flushDataToFile.

private void flushDataToFile(String destination, File dataFile) {
    FileMetaInstanceData data = new FileMetaInstanceData();
    if (destinations.containsKey(destination)) {
        synchronized (destination.intern()) {
            // 基于destination控制一下并发更新
            data.setDestination(destination);
            List<FileMetaClientIdentityData> clientDatas = Lists.newArrayList();
            List<ClientIdentity> clientIdentitys = destinations.get(destination);
            for (ClientIdentity clientIdentity : clientIdentitys) {
                FileMetaClientIdentityData clientData = new FileMetaClientIdentityData();
                clientData.setClientIdentity(clientIdentity);
                Position position = cursors.get(clientIdentity);
                if (position != null && position != nullCursor) {
                    clientData.setCursor((LogPosition) position);
                }
                clientDatas.add(clientData);
            }
            data.setClientDatas(clientDatas);
        }
        String json = JsonUtils.marshalToString(data);
        try {
            FileUtils.writeStringToFile(dataFile, json);
        } catch (IOException e) {
            throw new CanalMetaManagerException(e);
        }
    }
}

Example 4 with ClientIdentity

use of com.alibaba.otter.canal.protocol.ClientIdentity in project canal by alibaba.

the class MixedMetaManager method start.

public void start() {
    super.start();
    Assert.notNull(zooKeeperMetaManager);
    if (!zooKeeperMetaManager.isStart()) {
        zooKeeperMetaManager.start();
    }
    executor = Executors.newFixedThreadPool(1);
    destinations = MigrateMap.makeComputingMap(new Function<String, List<ClientIdentity>>() {

        public List<ClientIdentity> apply(String destination) {
            return zooKeeperMetaManager.listAllSubscribeInfo(destination);
        }
    });
    cursors = MigrateMap.makeComputingMap(new Function<ClientIdentity, Position>() {

        public Position apply(ClientIdentity clientIdentity) {
            Position position = zooKeeperMetaManager.getCursor(clientIdentity);
            if (position == null) {
                // 返回一个空对象标识，避免出现异常
                return nullCursor;
            } else {
                return position;
            }
        }
    });
    batches = MigrateMap.makeComputingMap(new Function<ClientIdentity, MemoryClientIdentityBatch>() {

        public MemoryClientIdentityBatch apply(ClientIdentity clientIdentity) {
            // 读取一下zookeeper信息，初始化一次
            MemoryClientIdentityBatch batches = MemoryClientIdentityBatch.create(clientIdentity);
            Map<Long, PositionRange> positionRanges = zooKeeperMetaManager.listAllBatchs(clientIdentity);
            for (Map.Entry<Long, PositionRange> entry : positionRanges.entrySet()) {
                // 添加记录到指定batchId
                batches.addPositionRange(entry.getValue(), entry.getKey());
            }
            return batches;
        }
    });
}

Example 5 with ClientIdentity

use of com.alibaba.otter.canal.protocol.ClientIdentity in project canal by alibaba.

the class PeriodMixedMetaManager method start.

public void start() {
    super.start();
    Assert.notNull(zooKeeperMetaManager);
    if (!zooKeeperMetaManager.isStart()) {
        zooKeeperMetaManager.start();
    }
    executor = Executors.newScheduledThreadPool(1);
    destinations = MigrateMap.makeComputingMap(new Function<String, List<ClientIdentity>>() {

        public List<ClientIdentity> apply(String destination) {
            return zooKeeperMetaManager.listAllSubscribeInfo(destination);
        }
    });
    cursors = MigrateMap.makeComputingMap(new Function<ClientIdentity, Position>() {

        public Position apply(ClientIdentity clientIdentity) {
            Position position = zooKeeperMetaManager.getCursor(clientIdentity);
            if (position == null) {
                // 返回一个空对象标识，避免出现异常
                return nullCursor;
            } else {
                return position;
            }
        }
    });
    batches = MigrateMap.makeComputingMap(new Function<ClientIdentity, MemoryClientIdentityBatch>() {

        public MemoryClientIdentityBatch apply(ClientIdentity clientIdentity) {
            // 读取一下zookeeper信息，初始化一次
            MemoryClientIdentityBatch batches = MemoryClientIdentityBatch.create(clientIdentity);
            Map<Long, PositionRange> positionRanges = zooKeeperMetaManager.listAllBatchs(clientIdentity);
            for (Map.Entry<Long, PositionRange> entry : positionRanges.entrySet()) {
                // 添加记录到指定batchId
                batches.addPositionRange(entry.getValue(), entry.getKey());
            }
            return batches;
        }
    });
    updateCursorTasks = Collections.synchronizedSet(new HashSet<ClientIdentity>());
    // 启动定时工作任务
    executor.scheduleAtFixedRate(new Runnable() {

        public void run() {
            List<ClientIdentity> tasks = new ArrayList<ClientIdentity>(updateCursorTasks);
            for (ClientIdentity clientIdentity : tasks) {
                try {
                    // 定时将内存中的最新值刷到zookeeper中，多次变更只刷一次
                    zooKeeperMetaManager.updateCursor(clientIdentity, getCursor(clientIdentity));
                    updateCursorTasks.remove(clientIdentity);
                } catch (Throwable e) {
                    // ignore
                    logger.error("period update" + clientIdentity.toString() + " curosr failed!", e);
                }
            }
        }
    }, period, period, TimeUnit.MILLISECONDS);
}