Examples with Position com.alibaba.otter.canal.protocol.position.Position used on opensource projects

Example 26 with Position

use of com.alibaba.otter.canal.protocol.position.Position in project canal by alibaba.

the class MemoryEventStorePutAndGetTest method testFullPutBatchGet.

@Test
public void testFullPutBatchGet() {
    int bufferSize = 16;
    MemoryEventStoreWithBuffer eventStore = new MemoryEventStoreWithBuffer();
    eventStore.setBufferSize(bufferSize);
    eventStore.start();
    for (int i = 0; i < bufferSize; i++) {
        boolean result = eventStore.tryPut(buildEvent("1", 1L, 1L + i));
        sleep(100L);
        Assert.assertTrue(result);
    }
    Position first = eventStore.getFirstPosition();
    Position lastest = eventStore.getLatestPosition();
    Assert.assertEquals(first, CanalEventUtils.createPosition(buildEvent("1", 1L, 1L)));
    Assert.assertEquals(lastest, CanalEventUtils.createPosition(buildEvent("1", 1L, 1L + bufferSize - 1)));
    System.out.println("start get");
    Events<Event> entrys1 = eventStore.tryGet(first, bufferSize);
    System.out.println("first get size : " + entrys1.getEvents().size());
    Assert.assertTrue(entrys1.getEvents().size() == bufferSize);
    Assert.assertEquals(first, entrys1.getPositionRange().getStart());
    Assert.assertEquals(lastest, entrys1.getPositionRange().getEnd());
    Assert.assertEquals(first, CanalEventUtils.createPosition(entrys1.getEvents().get(0)));
    Assert.assertEquals(lastest, CanalEventUtils.createPosition(entrys1.getEvents().get(bufferSize - 1)));
    eventStore.stop();
}

Example 27 with Position

use of com.alibaba.otter.canal.protocol.position.Position in project canal by alibaba.

the class FileMixedMetaManager method start.

public void start() {
    super.start();
    Assert.notNull(dataDir);
    if (!dataDir.exists()) {
        try {
            FileUtils.forceMkdir(dataDir);
        } catch (IOException e) {
            throw new CanalMetaManagerException(e);
        }
    }
    if (!dataDir.canRead() || !dataDir.canWrite()) {
        throw new CanalMetaManagerException("dir[" + dataDir.getPath() + "] can not read/write");
    }
    dataFileCaches = MigrateMap.makeComputingMap(this::getDataFile);
    executor = Executors.newScheduledThreadPool(1);
    destinations = MigrateMap.makeComputingMap(this::loadClientIdentity);
    cursors = MigrateMap.makeComputingMap(clientIdentity -> {
        Position position = loadCursor(clientIdentity.getDestination(), clientIdentity);
        if (position == null) {
            // 返回一个空对象标识，避免出现异常
            return nullCursor;
        } else {
            return position;
        }
    });
    updateCursorTasks = Collections.synchronizedSet(new HashSet<>());
    // 启动定时工作任务
    executor.scheduleAtFixedRate(() -> {
        List<ClientIdentity> tasks = new ArrayList<>(updateCursorTasks);
        for (ClientIdentity clientIdentity : tasks) {
            MDC.put("destination", String.valueOf(clientIdentity.getDestination()));
            try {
                updateCursorTasks.remove(clientIdentity);
                // 定时将内存中的最新值刷到file中，多次变更只刷一次
                if (logger.isInfoEnabled()) {
                    LogPosition cursor = (LogPosition) getCursor(clientIdentity);
                    logger.info("clientId:{} cursor:[{},{},{},{},{}] address[{}]", clientIdentity.getClientId(), cursor.getPostion().getJournalName(), cursor.getPostion().getPosition(), cursor.getPostion().getTimestamp(), cursor.getPostion().getServerId(), cursor.getPostion().getGtid(), cursor.getIdentity().getSourceAddress().toString());
                }
                flushDataToFile(clientIdentity.getDestination());
            } catch (Throwable e) {
                // ignore
                logger.error("period update" + clientIdentity.toString() + " curosr failed!", e);
            }
        }
    }, period, period, TimeUnit.MILLISECONDS);
}

Example 28 with Position

use of com.alibaba.otter.canal.protocol.position.Position in project canal by alibaba.

the class AbstractMetaManagerTest method doCursorTest.

public Position doCursorTest(CanalMetaManager metaManager) {
    metaManager.subscribe(clientIdentity);
    Position position1 = metaManager.getCursor(clientIdentity);
    Assert.assertNull(position1);
    PositionRange range = buildRange(1);
    metaManager.updateCursor(clientIdentity, range.getStart());
    Position position2 = metaManager.getCursor(clientIdentity);
    Assert.assertEquals(range.getStart(), position2);
    metaManager.updateCursor(clientIdentity, range.getEnd());
    Position position3 = metaManager.getCursor(clientIdentity);
    Assert.assertEquals(range.getEnd(), position3);
    return position3;
}

Example 29 with Position

use of com.alibaba.otter.canal.protocol.position.Position in project canal by alibaba.

the class FileMixedMetaManagerTest method testCursorAll.

@Test
public void testCursorAll() {
    FileMixedMetaManager metaManager = new FileMixedMetaManager();
    metaManager.setDataDirByFile(dataDir);
    metaManager.setPeriod(100);
    metaManager.start();
    Position lastPosition = doCursorTest(metaManager);
    sleep(1000L);
    // 重新构建一次，能获得上一次zk上的记录
    FileMixedMetaManager metaManager2 = new FileMixedMetaManager();
    metaManager2.setDataDirByFile(dataDir);
    metaManager2.setPeriod(100);
    metaManager2.start();
    Position position = metaManager2.getCursor(clientIdentity);
    Assert.assertEquals(position, lastPosition);
    metaManager.stop();
}

Example 30 with Position

use of com.alibaba.otter.canal.protocol.position.Position in project canal by alibaba.

the class MixedMetaManagerTest method testCursorAll.

@Test
public void testCursorAll() {
    MixedMetaManager metaManager = new MixedMetaManager();
    ZooKeeperMetaManager zooKeeperMetaManager = new ZooKeeperMetaManager();
    zooKeeperMetaManager.setZkClientx(zkclientx);
    metaManager.setZooKeeperMetaManager(zooKeeperMetaManager);
    metaManager.start();
    Position lastPosition = doCursorTest(metaManager);
    sleep(1000L);
    // 重新构建一次，能获得上一次zk上的记录
    MixedMetaManager metaManager2 = new MixedMetaManager();
    metaManager2.setZooKeeperMetaManager(zooKeeperMetaManager);
    metaManager2.start();
    Position position = metaManager2.getCursor(clientIdentity);
    Assert.assertEquals(position, lastPosition);
    metaManager.stop();
}