Examples with LogPosition com.alibaba.otter.canal.protocol.position.LogPosition used on opensource projects

Example 41 with LogPosition

use of com.alibaba.otter.canal.protocol.position.LogPosition in project canal by alibaba.

the class FileMixedMetaManager method start.

public void start() {
    super.start();
    Assert.notNull(dataDir);
    if (!dataDir.exists()) {
        try {
            FileUtils.forceMkdir(dataDir);
        } catch (IOException e) {
            throw new CanalMetaManagerException(e);
        }
    }
    if (!dataDir.canRead() || !dataDir.canWrite()) {
        throw new CanalMetaManagerException("dir[" + dataDir.getPath() + "] can not read/write");
    }
    dataFileCaches = MigrateMap.makeComputingMap(this::getDataFile);
    executor = Executors.newScheduledThreadPool(1);
    destinations = MigrateMap.makeComputingMap(this::loadClientIdentity);
    cursors = MigrateMap.makeComputingMap(clientIdentity -> {
        Position position = loadCursor(clientIdentity.getDestination(), clientIdentity);
        if (position == null) {
            // 返回一个空对象标识，避免出现异常
            return nullCursor;
        } else {
            return position;
        }
    });
    updateCursorTasks = Collections.synchronizedSet(new HashSet<>());
    // 启动定时工作任务
    executor.scheduleAtFixedRate(() -> {
        List<ClientIdentity> tasks = new ArrayList<>(updateCursorTasks);
        for (ClientIdentity clientIdentity : tasks) {
            MDC.put("destination", String.valueOf(clientIdentity.getDestination()));
            try {
                updateCursorTasks.remove(clientIdentity);
                // 定时将内存中的最新值刷到file中，多次变更只刷一次
                if (logger.isInfoEnabled()) {
                    LogPosition cursor = (LogPosition) getCursor(clientIdentity);
                    logger.info("clientId:{} cursor:[{},{},{},{},{}] address[{}]", clientIdentity.getClientId(), cursor.getPostion().getJournalName(), cursor.getPostion().getPosition(), cursor.getPostion().getTimestamp(), cursor.getPostion().getServerId(), cursor.getPostion().getGtid(), cursor.getIdentity().getSourceAddress().toString());
                }
                flushDataToFile(clientIdentity.getDestination());
            } catch (Throwable e) {
                // ignore
                logger.error("period update" + clientIdentity.toString() + " curosr failed!", e);
            }
        }
    }, period, period, TimeUnit.MILLISECONDS);
}

Example 42 with LogPosition

use of com.alibaba.otter.canal.protocol.position.LogPosition in project canal by alibaba.

the class AbstractEventParser method buildLastPosition.

protected LogPosition buildLastPosition(CanalEntry.Entry entry) {
    // 初始化一下
    LogPosition logPosition = new LogPosition();
    EntryPosition position = new EntryPosition();
    position.setJournalName(entry.getHeader().getLogfileName());
    position.setPosition(entry.getHeader().getLogfileOffset());
    position.setTimestamp(entry.getHeader().getExecuteTime());
    // add serverId at 2016-06-28
    position.setServerId(entry.getHeader().getServerId());
    // set gtid
    position.setGtid(entry.getHeader().getGtid());
    logPosition.setPostion(position);
    LogIdentity identity = new LogIdentity(runningInfo.getAddress(), -1L);
    logPosition.setIdentity(identity);
    return logPosition;
}

Example 43 with LogPosition

use of com.alibaba.otter.canal.protocol.position.LogPosition in project canal by alibaba.

the class CanalServerWithEmbedded method get.

/**
 * 获取数据，可以指定超时时间.
 *
 * <pre>
 * 几种case:
 * a. 如果timeout为null，则采用tryGet方式，即时获取
 * b. 如果timeout不为null
 *    1. timeout为0，则采用get阻塞方式，获取数据，不设置超时，直到有足够的batchSize数据才返回
 *    2. timeout不为0，则采用get+timeout方式，获取数据，超时还没有batchSize足够的数据，有多少返回多少
 *
 * 注意： meta获取和数据的获取需要保证顺序性，优先拿到meta的，一定也会是优先拿到数据，所以需要加同步. (不能出现先拿到meta，拿到第二批数据，这样就会导致数据顺序性出现问题)
 * </pre>
 */
@Override
public Message get(ClientIdentity clientIdentity, int batchSize, Long timeout, TimeUnit unit) throws CanalServerException {
    checkStart(clientIdentity.getDestination());
    checkSubscribe(clientIdentity);
    CanalInstance canalInstance = canalInstances.get(clientIdentity.getDestination());
    synchronized (canalInstance) {
        // 获取到流式数据中的最后一批获取的位置
        PositionRange<LogPosition> positionRanges = canalInstance.getMetaManager().getLastestBatch(clientIdentity);
        if (positionRanges != null) {
            throw new CanalServerException(String.format("clientId:%s has last batch:[%s] isn't ack , maybe loss data", clientIdentity.getClientId(), positionRanges));
        }
        Events<Event> events = null;
        Position start = canalInstance.getMetaManager().getCursor(clientIdentity);
        events = getEvents(canalInstance.getEventStore(), start, batchSize, timeout, unit);
        if (CollectionUtils.isEmpty(events.getEvents())) {
            logger.debug("get successfully, clientId:{} batchSize:{} but result is null", clientIdentity.getClientId(), batchSize);
            // 返回空包，避免生成batchId，浪费性能
            return new Message(-1, true, new ArrayList());
        } else {
            // 记录到流式信息
            Long batchId = canalInstance.getMetaManager().addBatch(clientIdentity, events.getPositionRange());
            boolean raw = isRaw(canalInstance.getEventStore());
            List entrys = null;
            if (raw) {
                entrys = Lists.transform(events.getEvents(), Event::getRawEntry);
            } else {
                entrys = Lists.transform(events.getEvents(), Event::getEntry);
            }
            if (logger.isInfoEnabled()) {
                logger.info("get successfully, clientId:{} batchSize:{} real size is {} and result is [batchId:{} , position:{}]", clientIdentity.getClientId(), batchSize, entrys.size(), batchId, events.getPositionRange());
            }
            // 直接提交ack
            ack(clientIdentity, batchId);
            return new Message(batchId, raw, entrys);
        }
    }
}

Example 44 with LogPosition

use of com.alibaba.otter.canal.protocol.position.LogPosition in project canal by alibaba.

the class AbstractCanalStoreScavenge method getLatestAckPosition.

/**
 * 找出该destination中可被清理掉的position位置
 *
 * @param destination
 */
private Position getLatestAckPosition(String destination) {
    List<ClientIdentity> clientIdentitys = canalMetaManager.listAllSubscribeInfo(destination);
    LogPosition result = null;
    if (!CollectionUtils.isEmpty(clientIdentitys)) {
        // 尝试找到一个最小的logPosition
        for (ClientIdentity clientIdentity : clientIdentitys) {
            LogPosition position = (LogPosition) canalMetaManager.getCursor(clientIdentity);
            if (position == null) {
                continue;
            }
            if (result == null) {
                result = position;
            } else {
                result = min(result, position);
            }
        }
    }
    return result;
}

Example 45 with LogPosition

use of com.alibaba.otter.canal.protocol.position.LogPosition in project canal by alibaba.

the class MemoryEventStoreWithBuffer method cleanUntil.

public void cleanUntil(Position position, Long seqId) throws CanalStoreException {
    final ReentrantLock lock = this.lock;
    lock.lock();
    try {
        long sequence = ackSequence.get();
        long maxSequence = getSequence.get();
        boolean hasMatch = false;
        long memsize = 0;
        // ack没有list，但有已存在的foreach，还是节省一下list的开销
        long localExecTime = 0L;
        int deltaRows = 0;
        if (seqId > 0) {
            maxSequence = seqId;
        }
        for (long next = sequence + 1; next <= maxSequence; next++) {
            Event event = entries[getIndex(next)];
            if (localExecTime == 0 && event.getExecuteTime() > 0) {
                localExecTime = event.getExecuteTime();
            }
            deltaRows += event.getRowsCount();
            memsize += calculateSize(event);
            if ((seqId < 0 || next == seqId) && CanalEventUtils.checkPosition(event, (LogPosition) position)) {
                // 找到对应的position，更新ack seq
                hasMatch = true;
                if (batchMode.isMemSize()) {
                    ackMemSize.addAndGet(memsize);
                    // 尝试清空buffer中的内存，将ack之前的内存全部释放掉
                    for (long index = sequence + 1; index < next; index++) {
                        // 设置为null
                        entries[getIndex(index)] = null;
                    }
                    // 考虑getFirstPosition/getLastPosition会获取最后一次ack的position信息
                    // ack清理的时候只处理entry=null，释放内存
                    Event lastEvent = entries[getIndex(next)];
                    lastEvent.setEntry(null);
                    lastEvent.setRawEntry(null);
                }
                if (ackSequence.compareAndSet(sequence, next)) {
                    // 避免并发ack
                    notFull.signal();
                    ackTableRows.addAndGet(deltaRows);
                    if (localExecTime > 0) {
                        ackExecTime.lazySet(localExecTime);
                    }
                    return;
                }
            }
        }
        if (!hasMatch) {
            // 找不到对应需要ack的position
            throw new CanalStoreException("no match ack position" + position.toString());
        }
    } finally {
        lock.unlock();
    }
}