Examples with PositionRange com.alibaba.otter.canal.protocol.position.PositionRange used on opensource projects

Example 6 with PositionRange

use of com.alibaba.otter.canal.protocol.position.PositionRange in project canal by alibaba.

the class ZooKeeperMetaManager method getLastestBatch.

public PositionRange getLastestBatch(ClientIdentity clientIdentity) {
    String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(), clientIdentity.getClientId());
    List<String> nodes = null;
    try {
        nodes = zkClientx.getChildren(path);
    } catch (ZkNoNodeException e) {
    // ignore
    }
    if (CollectionUtils.isEmpty(nodes)) {
        return null;
    }
    // 找到最大的Id
    ArrayList<Long> batchIds = new ArrayList<Long>(nodes.size());
    for (String batchIdString : nodes) {
        batchIds.add(Long.valueOf(batchIdString));
    }
    Long maxBatchId = Collections.max(batchIds);
    PositionRange result = getBatch(clientIdentity, maxBatchId);
    if (result == null) {
        // 出现为null，说明zk节点有变化，重新获取
        return getLastestBatch(clientIdentity);
    } else {
        return result;
    }
}

Example 7 with PositionRange

use of com.alibaba.otter.canal.protocol.position.PositionRange in project canal by alibaba.

the class AbstractMetaManagerTest method buildRange.

private PositionRange<LogPosition> buildRange(int number) {
    LogPosition start = new LogPosition();
    start.setIdentity(new LogIdentity(new InetSocketAddress(MYSQL_ADDRESS, 3306), 1234L));
    start.setPostion(new EntryPosition("mysql-bin.000000" + number, 106L, new Date().getTime()));
    LogPosition end = new LogPosition();
    end.setIdentity(new LogIdentity(new InetSocketAddress(MYSQL_ADDRESS, 3306), 1234L));
    end.setPostion(new EntryPosition("mysql-bin.000000" + (number + 1), 106L, (new Date().getTime()) + 1000 * 1000L));
    return new PositionRange<LogPosition>(start, end);
}

Example 8 with PositionRange

use of com.alibaba.otter.canal.protocol.position.PositionRange in project canal by alibaba.

the class MemoryEventStoreWithBuffer method doGet.

private Events<Event> doGet(Position start, int batchSize) throws CanalStoreException {
    LogPosition startPosition = (LogPosition) start;
    long current = getSequence.get();
    long maxAbleSequence = putSequence.get();
    long next = current;
    long end = current;
    // 如果startPosition为null，说明是第一次，默认+1处理
    if (startPosition == null || !startPosition.getPostion().isIncluded()) {
        // 第一次订阅之后，需要包含一下start位置，防止丢失第一条记录
        next = next + 1;
    }
    if (current >= maxAbleSequence) {
        return new Events<Event>();
    }
    Events<Event> result = new Events<Event>();
    List<Event> entrys = result.getEvents();
    long memsize = 0;
    if (batchMode.isItemSize()) {
        end = (next + batchSize - 1) < maxAbleSequence ? (next + batchSize - 1) : maxAbleSequence;
        // 提取数据并返回
        for (; next <= end; next++) {
            Event event = entries[getIndex(next)];
            if (ddlIsolation && isDdl(event.getEntry().getHeader().getEventType())) {
                // 如果是ddl隔离，直接返回
                if (entrys.size() == 0) {
                    // 如果没有DML事件，加入当前的DDL事件
                    entrys.add(event);
                    // 更新end为当前
                    end = next;
                } else {
                    // 如果之前已经有DML事件，直接返回了，因为不包含当前next这记录，需要回退一个位置
                    // next-1一定大于current，不需要判断
                    end = next - 1;
                }
                break;
            } else {
                entrys.add(event);
            }
        }
    } else {
        long maxMemSize = batchSize * bufferMemUnit;
        for (; memsize <= maxMemSize && next <= maxAbleSequence; next++) {
            // 永远保证可以取出第一条的记录，避免死锁
            Event event = entries[getIndex(next)];
            if (ddlIsolation && isDdl(event.getEntry().getHeader().getEventType())) {
                // 如果是ddl隔离，直接返回
                if (entrys.size() == 0) {
                    // 如果没有DML事件，加入当前的DDL事件
                    entrys.add(event);
                    // 更新end为当前
                    end = next;
                } else {
                    // 如果之前已经有DML事件，直接返回了，因为不包含当前next这记录，需要回退一个位置
                    // next-1一定大于current，不需要判断
                    end = next - 1;
                }
                break;
            } else {
                entrys.add(event);
                memsize += calculateSize(event);
                // 记录end位点
                end = next;
            }
        }
    }
    PositionRange<LogPosition> range = new PositionRange<LogPosition>();
    result.setPositionRange(range);
    range.setStart(CanalEventUtils.createPosition(entrys.get(0)));
    range.setEnd(CanalEventUtils.createPosition(entrys.get(result.getEvents().size() - 1)));
    for (int i = entrys.size() - 1; i >= 0; i--) {
        Event event = entrys.get(i);
        if (CanalEntry.EntryType.TRANSACTIONBEGIN == event.getEntry().getEntryType() || CanalEntry.EntryType.TRANSACTIONEND == event.getEntry().getEntryType() || isDdl(event.getEntry().getHeader().getEventType())) {
            // 将事务头/尾设置可被为ack的点
            range.setAck(CanalEventUtils.createPosition(event));
            break;
        }
    }
    if (getSequence.compareAndSet(current, end)) {
        getMemSize.addAndGet(memsize);
        notFull.signal();
        return result;
    } else {
        return new Events<Event>();
    }
}

Example 9 with PositionRange

use of com.alibaba.otter.canal.protocol.position.PositionRange in project canal by alibaba.

the class ZooKeeperMetaManager method getBatch.

public PositionRange getBatch(ClientIdentity clientIdentity, Long batchId) throws CanalMetaManagerException {
    String path = ZookeeperPathUtils.getBatchMarkWithIdPath(clientIdentity.getDestination(), clientIdentity.getClientId(), batchId);
    byte[] data = zkClientx.readData(path, true);
    if (data == null) {
        return null;
    }
    PositionRange positionRange = JsonUtils.unmarshalFromByte(data, PositionRange.class);
    return positionRange;
}

Example 10 with PositionRange

use of com.alibaba.otter.canal.protocol.position.PositionRange in project canal by alibaba.

the class ZooKeeperMetaManager method getFirstBatch.

public PositionRange getFirstBatch(ClientIdentity clientIdentity) {
    String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(), clientIdentity.getClientId());
    List<String> nodes = null;
    try {
        nodes = zkClientx.getChildren(path);
    } catch (ZkNoNodeException e) {
    // ignore
    }
    if (CollectionUtils.isEmpty(nodes)) {
        return null;
    }
    // 找到最小的Id
    ArrayList<Long> batchIds = new ArrayList<Long>(nodes.size());
    for (String batchIdString : nodes) {
        batchIds.add(Long.valueOf(batchIdString));
    }
    Long minBatchId = Collections.min(batchIds);
    PositionRange result = getBatch(clientIdentity, minBatchId);
    if (result == null) {
        // 出现为null，说明zk节点有变化，重新获取
        return getFirstBatch(clientIdentity);
    } else {
        return result;
    }
}