Examples with LogPosition com.alibaba.otter.canal.protocol.position.LogPosition used on opensource projects

Example 21 with LogPosition

use of com.alibaba.otter.canal.protocol.position.LogPosition in project canal by alibaba.

the class CanalServerWithEmbedded method ack.

/**
 * 进行 batch id 的确认。确认之后，小于等于此 batchId 的 Message 都会被确认。
 *
 * <pre>
 * 注意：进行反馈时必须按照batchId的顺序进行ack(需有客户端保证)
 * </pre>
 */
@Override
public void ack(ClientIdentity clientIdentity, long batchId) throws CanalServerException {
    checkStart(clientIdentity.getDestination());
    checkSubscribe(clientIdentity);
    CanalInstance canalInstance = canalInstances.get(clientIdentity.getDestination());
    PositionRange<LogPosition> positionRanges = null;
    // 更新位置
    positionRanges = canalInstance.getMetaManager().removeBatch(clientIdentity, batchId);
    if (positionRanges == null) {
        // 说明是重复的ack/rollback
        throw new CanalServerException(String.format("ack error , clientId:%s batchId:%d is not exist , please check", clientIdentity.getClientId(), batchId));
    }
    // 更新cursor
    if (positionRanges.getAck() != null) {
        canalInstance.getMetaManager().updateCursor(clientIdentity, positionRanges.getAck());
        if (logger.isInfoEnabled()) {
            logger.info("ack successfully, clientId:{} batchId:{} position:{}", clientIdentity.getClientId(), batchId, positionRanges);
        }
    }
    // 可定时清理数据
    canalInstance.getEventStore().ack(positionRanges.getEnd(), positionRanges.getEndSeq());
}

Example 22 with LogPosition

use of com.alibaba.otter.canal.protocol.position.LogPosition in project canal by alibaba.

the class MemoryEventStoreWithBuffer method doGet.

private Events<Event> doGet(Position start, int batchSize) throws CanalStoreException {
    LogPosition startPosition = (LogPosition) start;
    long current = getSequence.get();
    long maxAbleSequence = putSequence.get();
    long next = current;
    long end = current;
    // 如果startPosition为null，说明是第一次，默认+1处理
    if (startPosition == null || !startPosition.getPostion().isIncluded()) {
        // 第一次订阅之后，需要包含一下start位置，防止丢失第一条记录
        next = next + 1;
    }
    if (current >= maxAbleSequence) {
        return new Events<>();
    }
    Events<Event> result = new Events<>();
    List<Event> entrys = result.getEvents();
    long memsize = 0;
    if (batchMode.isItemSize()) {
        end = (next + batchSize - 1) < maxAbleSequence ? (next + batchSize - 1) : maxAbleSequence;
        // 提取数据并返回
        for (; next <= end; next++) {
            Event event = entries[getIndex(next)];
            if (ddlIsolation && isDdl(event.getEventType())) {
                // 如果是ddl隔离，直接返回
                if (entrys.size() == 0) {
                    // 如果没有DML事件，加入当前的DDL事件
                    entrys.add(event);
                    // 更新end为当前
                    end = next;
                } else {
                    // 如果之前已经有DML事件，直接返回了，因为不包含当前next这记录，需要回退一个位置
                    // next-1一定大于current，不需要判断
                    end = next - 1;
                }
                break;
            } else {
                entrys.add(event);
            }
        }
    } else {
        long maxMemSize = batchSize * bufferMemUnit;
        for (; memsize <= maxMemSize && next <= maxAbleSequence; next++) {
            // 永远保证可以取出第一条的记录，避免死锁
            Event event = entries[getIndex(next)];
            if (ddlIsolation && isDdl(event.getEventType())) {
                // 如果是ddl隔离，直接返回
                if (entrys.size() == 0) {
                    // 如果没有DML事件，加入当前的DDL事件
                    entrys.add(event);
                    // 更新end为当前
                    end = next;
                } else {
                    // 如果之前已经有DML事件，直接返回了，因为不包含当前next这记录，需要回退一个位置
                    // next-1一定大于current，不需要判断
                    end = next - 1;
                }
                break;
            } else {
                entrys.add(event);
                memsize += calculateSize(event);
                // 记录end位点
                end = next;
            }
        }
    }
    PositionRange<LogPosition> range = new PositionRange<>();
    result.setPositionRange(range);
    range.setStart(CanalEventUtils.createPosition(entrys.get(0)));
    range.setEnd(CanalEventUtils.createPosition(entrys.get(result.getEvents().size() - 1)));
    range.setEndSeq(end);
    for (int i = entrys.size() - 1; i >= 0; i--) {
        Event event = entrys.get(i);
        // GTID模式,ack的位点必须是事务结尾,因为下一次订阅的时候mysql会发送这个gtid之后的next,如果在事务头就记录了会丢这最后一个事务
        if ((CanalEntry.EntryType.TRANSACTIONBEGIN == event.getEntryType() && StringUtils.isEmpty(event.getGtid())) || CanalEntry.EntryType.TRANSACTIONEND == event.getEntryType() || isDdl(event.getEventType())) {
            // 将事务头/尾设置可被为ack的点
            range.setAck(CanalEventUtils.createPosition(event));
            break;
        }
    }
    if (getSequence.compareAndSet(current, end)) {
        getMemSize.addAndGet(memsize);
        notFull.signal();
        profiling(result.getEvents(), OP.GET);
        return result;
    } else {
        return new Events<>();
    }
}

Example 23 with LogPosition

use of com.alibaba.otter.canal.protocol.position.LogPosition in project canal by alibaba.

the class CanalEventUtils method createPosition.

/**
 * 根据entry创建对应的Position对象
 */
public static LogPosition createPosition(Event event, boolean included) {
    EntryPosition position = new EntryPosition();
    position.setJournalName(event.getJournalName());
    position.setPosition(event.getPosition());
    position.setTimestamp(event.getExecuteTime());
    position.setIncluded(included);
    // add serverId at 2016-06-28
    position.setServerId(event.getServerId());
    // add gtid
    position.setGtid(event.getGtid());
    LogPosition logPosition = new LogPosition();
    logPosition.setPostion(position);
    logPosition.setIdentity(event.getLogIdentity());
    return logPosition;
}

Example 24 with LogPosition

use of com.alibaba.otter.canal.protocol.position.LogPosition in project canal by alibaba.

the class PeriodMixedLogPositionManager method start.

public void start() {
    super.start();
    Assert.notNull(zooKeeperLogPositionManager);
    if (!zooKeeperLogPositionManager.isStart()) {
        zooKeeperLogPositionManager.start();
    }
    executor = Executors.newScheduledThreadPool(1);
    positions = MigrateMap.makeComputingMap(new Function<String, LogPosition>() {

        public LogPosition apply(String destination) {
            LogPosition logPosition = zooKeeperLogPositionManager.getLatestIndexBy(destination);
            if (logPosition == null) {
                return nullPosition;
            } else {
                return logPosition;
            }
        }
    });
    persistTasks = Collections.synchronizedSet(new HashSet<String>());
    // 启动定时工作任务
    executor.scheduleAtFixedRate(new Runnable() {

        public void run() {
            List<String> tasks = new ArrayList<String>(persistTasks);
            for (String destination : tasks) {
                try {
                    // 定时将内存中的最新值刷到zookeeper中，多次变更只刷一次
                    zooKeeperLogPositionManager.persistLogPosition(destination, getLatestIndexBy(destination));
                    persistTasks.remove(destination);
                } catch (Throwable e) {
                    // ignore
                    logger.error("period update" + destination + " curosr failed!", e);
                }
            }
        }
    }, period, period, TimeUnit.MILLISECONDS);
}

Example 25 with LogPosition

use of com.alibaba.otter.canal.protocol.position.LogPosition in project canal by alibaba.

the class MysqlEventParser method findStartPositionInternal.

protected EntryPosition findStartPositionInternal(ErosaConnection connection) {
    MysqlConnection mysqlConnection = (MysqlConnection) connection;
    LogPosition logPosition = logPositionManager.getLatestIndexBy(destination);
    if (logPosition == null) {
        // 找不到历史成功记录
        EntryPosition entryPosition = null;
        if (masterInfo != null && mysqlConnection.getConnector().getAddress().equals(masterInfo.getAddress())) {
            entryPosition = masterPosition;
        } else if (standbyInfo != null && mysqlConnection.getConnector().getAddress().equals(standbyInfo.getAddress())) {
            entryPosition = standbyPosition;
        }
        if (entryPosition == null) {
            // 默认从当前最后一个位置进行消费
            entryPosition = findEndPositionWithMasterIdAndTimestamp(mysqlConnection);
        }
        // 判断一下是否需要按时间订阅
        if (StringUtils.isEmpty(entryPosition.getJournalName())) {
            // 如果没有指定binlogName，尝试按照timestamp进行查找
            if (entryPosition.getTimestamp() != null && entryPosition.getTimestamp() > 0L) {
                logger.warn("prepare to find start position {}:{}:{}", new Object[] { "", "", entryPosition.getTimestamp() });
                return findByStartTimeStamp(mysqlConnection, entryPosition.getTimestamp());
            } else {
                logger.warn("prepare to find start position just show master status");
                // 默认从当前最后一个位置进行消费
                return findEndPositionWithMasterIdAndTimestamp(mysqlConnection);
            }
        } else {
            if (entryPosition.getPosition() != null && entryPosition.getPosition() > 0L) {
                // 如果指定binlogName + offest，直接返回
                entryPosition = findPositionWithMasterIdAndTimestamp(mysqlConnection, entryPosition);
                logger.warn("prepare to find start position {}:{}:{}", new Object[] { entryPosition.getJournalName(), entryPosition.getPosition(), entryPosition.getTimestamp() });
                return entryPosition;
            } else {
                EntryPosition specificLogFilePosition = null;
                if (entryPosition.getTimestamp() != null && entryPosition.getTimestamp() > 0L) {
                    // 如果指定binlogName +
                    // timestamp，但没有指定对应的offest，尝试根据时间找一下offest
                    EntryPosition endPosition = findEndPosition(mysqlConnection);
                    if (endPosition != null) {
                        logger.warn("prepare to find start position {}:{}:{}", new Object[] { entryPosition.getJournalName(), "", entryPosition.getTimestamp() });
                        specificLogFilePosition = findAsPerTimestampInSpecificLogFile(mysqlConnection, entryPosition.getTimestamp(), endPosition, entryPosition.getJournalName(), true);
                    }
                }
                if (specificLogFilePosition == null) {
                    if (isRdsOssMode()) {
                        // 如果binlog位点不存在，并且属于timestamp不为空,可以返回null走到oss binlog处理
                        return null;
                    }
                    // position不存在，从文件头开始
                    entryPosition.setPosition(BINLOG_START_OFFEST);
                    return entryPosition;
                } else {
                    return specificLogFilePosition;
                }
            }
        }
    } else {
        if (logPosition.getIdentity().getSourceAddress().equals(mysqlConnection.getConnector().getAddress())) {
            if (dumpErrorCountThreshold >= 0 && dumpErrorCount > dumpErrorCountThreshold) {
                // binlog定位位点失败,可能有两个原因:
                // 1. binlog位点被删除
                // 2.vip模式的mysql,发生了主备切换,判断一下serverId是否变化,针对这种模式可以发起一次基于时间戳查找合适的binlog位点
                boolean case2 = (standbyInfo == null || standbyInfo.getAddress() == null) && logPosition.getPostion().getServerId() != null && !logPosition.getPostion().getServerId().equals(findServerId(mysqlConnection));
                if (case2) {
                    EntryPosition findPosition = fallbackFindByStartTimestamp(logPosition, mysqlConnection);
                    dumpErrorCount = 0;
                    return findPosition;
                }
                // 丢失调到最新位点也即意味着数据丢失
                if (isAutoResetLatestPosMode()) {
                    dumpErrorCount = 0;
                    return findEndPosition(mysqlConnection);
                }
                Long timestamp = logPosition.getPostion().getTimestamp();
                if (isRdsOssMode() && (timestamp != null && timestamp > 0)) {
                    // 如果binlog位点不存在，并且属于timestamp不为空,可以返回null走到oss binlog处理
                    return null;
                }
            } else if (StringUtils.isBlank(logPosition.getPostion().getJournalName()) && logPosition.getPostion().getPosition() <= 0 && logPosition.getPostion().getTimestamp() > 0) {
                return fallbackFindByStartTimestamp(logPosition, mysqlConnection);
            }
            // 其余情况
            logger.warn("prepare to find start position just last position\n {}", JsonUtils.marshalToString(logPosition));
            return logPosition.getPostion();
        } else {
            // 针对切换的情况，考虑回退时间
            long newStartTimestamp = logPosition.getPostion().getTimestamp() - fallbackIntervalInSeconds * 1000;
            logger.warn("prepare to find start position by switch {}:{}:{}", new Object[] { "", "", logPosition.getPostion().getTimestamp() });
            return findByStartTimeStamp(mysqlConnection, newStartTimestamp);
        }
    }
}