Examples with LogPosition com.alibaba.otter.canal.protocol.position.LogPosition used on opensource projects

Example 16 with LogPosition

use of com.alibaba.otter.canal.protocol.position.LogPosition in project canal by alibaba.

the class RdsLocalBinlogEventParser method onFinish.

@Override
public void onFinish(String fileName) {
    try {
        binlogDownloadQueue.downOne();
        File needDeleteFile = new File(directory + File.separator + fileName);
        if (needDeleteFile.exists()) {
            needDeleteFile.delete();
        }
        // 处理下logManager位点问题
        LogPosition logPosition = logPositionManager.getLatestIndexBy(destination);
        Long timestamp = 0L;
        if (logPosition != null && logPosition.getPostion() != null) {
            timestamp = logPosition.getPostion().getTimestamp();
            EntryPosition position = logPosition.getPostion();
            LogPosition newLogPosition = new LogPosition();
            String journalName = position.getJournalName();
            int sepIdx = journalName.indexOf(".");
            String fileIndex = journalName.substring(sepIdx + 1);
            int index = NumberUtils.toInt(fileIndex) + 1;
            String newJournalName = journalName.substring(0, sepIdx) + "." + StringUtils.leftPad(String.valueOf(index), fileIndex.length(), "0");
            newLogPosition.setPostion(new EntryPosition(newJournalName, 4L, position.getTimestamp(), position.getServerId()));
            newLogPosition.setIdentity(logPosition.getIdentity());
            logPositionManager.persistLogPosition(destination, newLogPosition);
        }
        if (binlogDownloadQueue.isLastFile(fileName)) {
            logger.warn("last file : " + fileName + " , timestamp : " + timestamp + " , all file parse complete, switch to mysql parser!");
            finishListener.onFinish();
            return;
        } else {
            logger.warn("parse local binlog file : " + fileName + " , timestamp : " + timestamp + " , try the next binlog !");
        }
        binlogDownloadQueue.prepare();
    } catch (Exception e) {
        logger.error("prepare download binlog file failed!", e);
        throw new RuntimeException(e);
    }
}

Example 17 with LogPosition

use of com.alibaba.otter.canal.protocol.position.LogPosition in project canal by alibaba.

the class MetaLogPositionManager method getLatestIndexBy.

@Override
public LogPosition getLatestIndexBy(String destination) {
    List<ClientIdentity> clientIdentities = metaManager.listAllSubscribeInfo(destination);
    LogPosition result = null;
    if (!CollectionUtils.isEmpty(clientIdentities)) {
        // 尝试找到一个最小的logPosition
        for (ClientIdentity clientIdentity : clientIdentities) {
            LogPosition position = (LogPosition) metaManager.getCursor(clientIdentity);
            if (position == null) {
                continue;
            }
            if (result == null) {
                result = position;
            } else {
                result = CanalEventUtils.min(result, position);
            }
        }
    }
    return result;
}

Example 18 with LogPosition

use of com.alibaba.otter.canal.protocol.position.LogPosition in project canal by alibaba.

the class FileMixedLogPositionManager method getLatestIndexBy.

@Override
public LogPosition getLatestIndexBy(String destination) {
    LogPosition logPosition = memoryLogPositionManager.getLatestIndexBy(destination);
    if (logPosition != null) {
        return logPosition;
    }
    logPosition = loadDataFromFile(dataFileCaches.get(destination));
    if (logPosition == null) {
        return nullPosition;
    }
    return logPosition;
}

Example 19 with LogPosition

use of com.alibaba.otter.canal.protocol.position.LogPosition in project canal by alibaba.

the class CanalServerWithEmbedded method rollback.

/**
 * 回滚到未进行 {@link #ack} 的地方，下次fetch的时候，可以从最后一个没有 {@link #ack} 的地方开始拿
 */
@Override
public void rollback(ClientIdentity clientIdentity, Long batchId) throws CanalServerException {
    checkStart(clientIdentity.getDestination());
    CanalInstance canalInstance = canalInstances.get(clientIdentity.getDestination());
    // 因为存在第一次链接时自动rollback的情况，所以需要忽略未订阅
    boolean hasSubscribe = canalInstance.getMetaManager().hasSubscribe(clientIdentity);
    if (!hasSubscribe) {
        return;
    }
    synchronized (canalInstance) {
        // 清除batch信息
        PositionRange<LogPosition> positionRanges = canalInstance.getMetaManager().removeBatch(clientIdentity, batchId);
        if (positionRanges == null) {
            // 说明是重复的ack/rollback
            throw new CanalServerException(String.format("rollback error, clientId:%s batchId:%d is not exist , please check", clientIdentity.getClientId(), batchId));
        }
        // lastRollbackPostions.put(clientIdentity,
        // positionRanges.getEnd());// 记录一下最后rollback的位置
        // TODO 后续rollback到指定的batchId位置
        // rollback
        canalInstance.getEventStore().rollback();
        // eventStore中的状态信息
        logger.info("rollback successfully, clientId:{} batchId:{} position:{}", clientIdentity.getClientId(), batchId, positionRanges);
    }
}

Example 20 with LogPosition

use of com.alibaba.otter.canal.protocol.position.LogPosition in project canal by alibaba.

the class CanalServerWithEmbedded method getWithoutAck.

/**
 * 不指定 position 获取事件。canal 会记住此 client 最新的 position。 <br/>
 * 如果是第一次 fetch，则会从 canal 中保存的最老一条数据开始输出。
 *
 * <pre>
 * 几种case:
 * a. 如果timeout为null，则采用tryGet方式，即时获取
 * b. 如果timeout不为null
 *    1. timeout为0，则采用get阻塞方式，获取数据，不设置超时，直到有足够的batchSize数据才返回
 *    2. timeout不为0，则采用get+timeout方式，获取数据，超时还没有batchSize足够的数据，有多少返回多少
 *
 * 注意： meta获取和数据的获取需要保证顺序性，优先拿到meta的，一定也会是优先拿到数据，所以需要加同步. (不能出现先拿到meta，拿到第二批数据，这样就会导致数据顺序性出现问题)
 * </pre>
 */
@Override
public Message getWithoutAck(ClientIdentity clientIdentity, int batchSize, Long timeout, TimeUnit unit) throws CanalServerException {
    checkStart(clientIdentity.getDestination());
    checkSubscribe(clientIdentity);
    CanalInstance canalInstance = canalInstances.get(clientIdentity.getDestination());
    synchronized (canalInstance) {
        // 获取到流式数据中的最后一批获取的位置
        PositionRange<LogPosition> positionRanges = canalInstance.getMetaManager().getLastestBatch(clientIdentity);
        Events<Event> events = null;
        if (positionRanges != null) {
            // 存在流数据
            events = getEvents(canalInstance.getEventStore(), positionRanges.getStart(), batchSize, timeout, unit);
        } else {
            // ack后第一次获取
            Position start = canalInstance.getMetaManager().getCursor(clientIdentity);
            if (start == null) {
                // 第一次，还没有过ack记录，则获取当前store中的第一条
                start = canalInstance.getEventStore().getFirstPosition();
            }
            events = getEvents(canalInstance.getEventStore(), start, batchSize, timeout, unit);
        }
        if (CollectionUtils.isEmpty(events.getEvents())) {
            // 返回空包，避免生成batchId，浪费性能
            return new Message(-1, true, new ArrayList());
        } else {
            // 记录到流式信息
            Long batchId = canalInstance.getMetaManager().addBatch(clientIdentity, events.getPositionRange());
            boolean raw = isRaw(canalInstance.getEventStore());
            List entrys = null;
            if (raw) {
                entrys = Lists.transform(events.getEvents(), Event::getRawEntry);
            } else {
                entrys = Lists.transform(events.getEvents(), Event::getEntry);
            }
            if (logger.isInfoEnabled()) {
                logger.info("getWithoutAck successfully, clientId:{} batchSize:{}  real size is {} and result is [batchId:{} , position:{}]", clientIdentity.getClientId(), batchSize, entrys.size(), batchId, events.getPositionRange());
            }
            return new Message(batchId, raw, entrys);
        }
    }
}