Examples with PositionRange com.alibaba.otter.canal.protocol.position.PositionRange used on opensource projects

Example 1 with PositionRange

use of com.alibaba.otter.canal.protocol.position.PositionRange in project canal by alibaba.

the class AbstractMetaManagerTest method doBatchTest.

public void doBatchTest(CanalMetaManager metaManager) {
    metaManager.subscribe(clientIdentity);
    PositionRange first = metaManager.getFirstBatch(clientIdentity);
    PositionRange lastest = metaManager.getLastestBatch(clientIdentity);
    Assert.assertNull(first);
    Assert.assertNull(lastest);
    PositionRange range1 = buildRange(1);
    Long batchId1 = metaManager.addBatch(clientIdentity, range1);
    PositionRange range2 = buildRange(2);
    Long batchId2 = metaManager.addBatch(clientIdentity, range2);
    Assert.assertEquals((batchId1.longValue() + 1), batchId2.longValue());
    // 验证get
    PositionRange getRange1 = metaManager.getBatch(clientIdentity, batchId1);
    Assert.assertEquals(range1, getRange1);
    PositionRange getRange2 = metaManager.getBatch(clientIdentity, batchId2);
    Assert.assertEquals(range2, getRange2);
    PositionRange range3 = buildRange(3);
    Long batchId3 = batchId2 + 1;
    metaManager.addBatch(clientIdentity, range3, batchId3);
    PositionRange range4 = buildRange(4);
    Long batchId4 = metaManager.addBatch(clientIdentity, range4);
    Assert.assertEquals((batchId3.longValue() + 1), batchId4.longValue());
    // 验证remove
    metaManager.removeBatch(clientIdentity, batchId1);
    range1 = metaManager.getBatch(clientIdentity, batchId1);
    Assert.assertNull(range1);
    // 验证first / lastest
    first = metaManager.getFirstBatch(clientIdentity);
    lastest = metaManager.getLastestBatch(clientIdentity);
    Assert.assertEquals(range2, first);
    Assert.assertEquals(range4, lastest);
    Map<Long, PositionRange> ranges = metaManager.listAllBatchs(clientIdentity);
    Assert.assertEquals(3, ranges.size());
}

Example 2 with PositionRange

use of com.alibaba.otter.canal.protocol.position.PositionRange in project canal by alibaba.

the class MixedMetaManager method start.

public void start() {
    super.start();
    Assert.notNull(zooKeeperMetaManager);
    if (!zooKeeperMetaManager.isStart()) {
        zooKeeperMetaManager.start();
    }
    executor = Executors.newFixedThreadPool(1);
    destinations = MigrateMap.makeComputingMap(new Function<String, List<ClientIdentity>>() {

        public List<ClientIdentity> apply(String destination) {
            return zooKeeperMetaManager.listAllSubscribeInfo(destination);
        }
    });
    cursors = MigrateMap.makeComputingMap(new Function<ClientIdentity, Position>() {

        public Position apply(ClientIdentity clientIdentity) {
            Position position = zooKeeperMetaManager.getCursor(clientIdentity);
            if (position == null) {
                // 返回一个空对象标识，避免出现异常
                return nullCursor;
            } else {
                return position;
            }
        }
    });
    batches = MigrateMap.makeComputingMap(new Function<ClientIdentity, MemoryClientIdentityBatch>() {

        public MemoryClientIdentityBatch apply(ClientIdentity clientIdentity) {
            // 读取一下zookeeper信息，初始化一次
            MemoryClientIdentityBatch batches = MemoryClientIdentityBatch.create(clientIdentity);
            Map<Long, PositionRange> positionRanges = zooKeeperMetaManager.listAllBatchs(clientIdentity);
            for (Map.Entry<Long, PositionRange> entry : positionRanges.entrySet()) {
                // 添加记录到指定batchId
                batches.addPositionRange(entry.getValue(), entry.getKey());
            }
            return batches;
        }
    });
}

Example 3 with PositionRange

use of com.alibaba.otter.canal.protocol.position.PositionRange in project canal by alibaba.

the class MixedMetaManager method removeBatch.

public PositionRange removeBatch(final ClientIdentity clientIdentity, final Long batchId) throws CanalMetaManagerException {
    PositionRange positionRange = super.removeBatch(clientIdentity, batchId);
    // 异步刷新
    executor.submit(new Runnable() {

        public void run() {
            zooKeeperMetaManager.removeBatch(clientIdentity, batchId);
        }
    });
    return positionRange;
}

Example 4 with PositionRange

use of com.alibaba.otter.canal.protocol.position.PositionRange in project canal by alibaba.

the class PeriodMixedMetaManager method start.

public void start() {
    super.start();
    Assert.notNull(zooKeeperMetaManager);
    if (!zooKeeperMetaManager.isStart()) {
        zooKeeperMetaManager.start();
    }
    executor = Executors.newScheduledThreadPool(1);
    destinations = MigrateMap.makeComputingMap(new Function<String, List<ClientIdentity>>() {

        public List<ClientIdentity> apply(String destination) {
            return zooKeeperMetaManager.listAllSubscribeInfo(destination);
        }
    });
    cursors = MigrateMap.makeComputingMap(new Function<ClientIdentity, Position>() {

        public Position apply(ClientIdentity clientIdentity) {
            Position position = zooKeeperMetaManager.getCursor(clientIdentity);
            if (position == null) {
                // 返回一个空对象标识，避免出现异常
                return nullCursor;
            } else {
                return position;
            }
        }
    });
    batches = MigrateMap.makeComputingMap(new Function<ClientIdentity, MemoryClientIdentityBatch>() {

        public MemoryClientIdentityBatch apply(ClientIdentity clientIdentity) {
            // 读取一下zookeeper信息，初始化一次
            MemoryClientIdentityBatch batches = MemoryClientIdentityBatch.create(clientIdentity);
            Map<Long, PositionRange> positionRanges = zooKeeperMetaManager.listAllBatchs(clientIdentity);
            for (Map.Entry<Long, PositionRange> entry : positionRanges.entrySet()) {
                // 添加记录到指定batchId
                batches.addPositionRange(entry.getValue(), entry.getKey());
            }
            return batches;
        }
    });
    updateCursorTasks = Collections.synchronizedSet(new HashSet<ClientIdentity>());
    // 启动定时工作任务
    executor.scheduleAtFixedRate(new Runnable() {

        public void run() {
            List<ClientIdentity> tasks = new ArrayList<ClientIdentity>(updateCursorTasks);
            for (ClientIdentity clientIdentity : tasks) {
                try {
                    // 定时将内存中的最新值刷到zookeeper中，多次变更只刷一次
                    zooKeeperMetaManager.updateCursor(clientIdentity, getCursor(clientIdentity));
                    updateCursorTasks.remove(clientIdentity);
                } catch (Throwable e) {
                    // ignore
                    logger.error("period update" + clientIdentity.toString() + " curosr failed!", e);
                }
            }
        }
    }, period, period, TimeUnit.MILLISECONDS);
}

Example 5 with PositionRange

use of com.alibaba.otter.canal.protocol.position.PositionRange in project canal by alibaba.

the class ZooKeeperMetaManager method listAllBatchs.

public Map<Long, PositionRange> listAllBatchs(ClientIdentity clientIdentity) {
    String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(), clientIdentity.getClientId());
    List<String> nodes = null;
    try {
        nodes = zkClientx.getChildren(path);
    } catch (ZkNoNodeException e) {
    // ignore
    }
    if (CollectionUtils.isEmpty(nodes)) {
        return Maps.newHashMap();
    }
    // 找到最大的Id
    ArrayList<Long> batchIds = new ArrayList<Long>(nodes.size());
    for (String batchIdString : nodes) {
        batchIds.add(Long.valueOf(batchIdString));
    }
    // 从小到大排序
    Collections.sort(batchIds);
    Map<Long, PositionRange> positionRanges = Maps.newLinkedHashMap();
    for (Long batchId : batchIds) {
        PositionRange result = getBatch(clientIdentity, batchId);
        if (result == null) {
            // 出现为null，说明zk节点有变化，重新获取
            return listAllBatchs(clientIdentity);
        } else {
            positionRanges.put(batchId, result);
        }
    }
    return positionRanges;
}