Examples with Position com.alibaba.otter.canal.protocol.position.Position used on opensource projects

Example 11 with Position

use of com.alibaba.otter.canal.protocol.position.Position in project canal by alibaba.

the class MemoryEventStoreMemBatchTest method testBlockPutOneGet.

@Test
public void testBlockPutOneGet() {
    final MemoryEventStoreWithBuffer eventStore = new MemoryEventStoreWithBuffer();
    eventStore.setBufferSize(16);
    eventStore.setBatchMode(BatchMode.MEMSIZE);
    eventStore.start();
    final int batchSize = 10;
    for (int i = 0; i < batchSize; i++) {
        boolean result = eventStore.tryPut(buildEvent("1", 1L, 1L));
        Assert.assertTrue(result);
    }
    final Position position = eventStore.getFirstPosition();
    try {
        Events<Event> entrys = eventStore.get(position, batchSize);
        Assert.assertTrue(entrys.getEvents().size() == batchSize);
        Assert.assertEquals(position, entrys.getPositionRange().getStart());
        Assert.assertEquals(position, entrys.getPositionRange().getEnd());
    } catch (CanalStoreException e) {
    } catch (InterruptedException e) {
    }
    ExecutorService executor = Executors.newFixedThreadPool(1);
    executor.submit(new Runnable() {

        public void run() {
            boolean result = false;
            try {
                eventStore.get(position, batchSize);
            } catch (CanalStoreException e) {
            } catch (InterruptedException e) {
                System.out.println("interrupt occured.");
                result = true;
            }
            Assert.assertTrue(result);
        }
    });
    try {
        Thread.sleep(1000L);
    } catch (InterruptedException e) {
        Assert.fail(e.getMessage());
    }
    executor.shutdownNow();
    try {
        Thread.sleep(1000L);
    } catch (InterruptedException e) {
        Assert.fail(e.getMessage());
    }
    eventStore.stop();
}

Example 12 with Position

use of com.alibaba.otter.canal.protocol.position.Position in project canal by alibaba.

the class MemoryEventStorePutAndGetTest method testOnePutOneGet.

@Test
public void testOnePutOneGet() {
    MemoryEventStoreWithBuffer eventStore = new MemoryEventStoreWithBuffer();
    eventStore.start();
    boolean result = eventStore.tryPut(buildEvent("1", 1L, 1L));
    Assert.assertTrue(result);
    Position position = eventStore.getFirstPosition();
    Events<Event> entrys = eventStore.tryGet(position, 1);
    Assert.assertTrue(entrys.getEvents().size() == 1);
    Assert.assertEquals(position, entrys.getPositionRange().getStart());
    Assert.assertEquals(position, entrys.getPositionRange().getEnd());
    eventStore.stop();
}

Example 13 with Position

use of com.alibaba.otter.canal.protocol.position.Position in project canal by alibaba.

the class MemoryEventStorePutAndGetTest method testBlockPutOneGet.

@Test
public void testBlockPutOneGet() {
    final MemoryEventStoreWithBuffer eventStore = new MemoryEventStoreWithBuffer();
    eventStore.start();
    final int batchSize = 10;
    for (int i = 0; i < batchSize; i++) {
        boolean result = eventStore.tryPut(buildEvent("1", 1L, 1L));
        Assert.assertTrue(result);
    }
    final Position position = eventStore.getFirstPosition();
    try {
        Events<Event> entrys = eventStore.get(position, batchSize);
        Assert.assertTrue(entrys.getEvents().size() == batchSize);
        Assert.assertEquals(position, entrys.getPositionRange().getStart());
        Assert.assertEquals(position, entrys.getPositionRange().getEnd());
    } catch (CanalStoreException e) {
    } catch (InterruptedException e) {
    }
    ExecutorService executor = Executors.newFixedThreadPool(1);
    executor.submit(new Runnable() {

        public void run() {
            boolean result = false;
            try {
                eventStore.get(position, batchSize);
            } catch (CanalStoreException e) {
            } catch (InterruptedException e) {
                System.out.println("interrupt occured.");
                result = true;
            }
            Assert.assertTrue(result);
        }
    });
    try {
        Thread.sleep(1000L);
    } catch (InterruptedException e) {
        Assert.fail(e.getMessage());
    }
    executor.shutdownNow();
    try {
        Thread.sleep(1000L);
    } catch (InterruptedException e) {
        Assert.fail(e.getMessage());
    }
    eventStore.stop();
}

Example 14 with Position

use of com.alibaba.otter.canal.protocol.position.Position in project canal by alibaba.

the class MemoryEventStoreRollbackAndAckTest method testRollback.

@Test
public void testRollback() {
    int bufferSize = 16;
    MemoryEventStoreWithBuffer eventStore = new MemoryEventStoreWithBuffer();
    eventStore.setBufferSize(bufferSize);
    eventStore.start();
    for (int i = 0; i < bufferSize / 2; i++) {
        boolean result = eventStore.tryPut(buildEvent("1", 1L, 1L + i));
        sleep(100L);
        Assert.assertTrue(result);
    }
    sleep(50L);
    Position first = eventStore.getFirstPosition();
    Position lastest = eventStore.getLatestPosition();
    Assert.assertEquals(first, CanalEventUtils.createPosition(buildEvent("1", 1L, 1L)));
    Assert.assertEquals(lastest, CanalEventUtils.createPosition(buildEvent("1", 1L, 1L + bufferSize / 2 - 1)));
    System.out.println("start get");
    Events<Event> entrys1 = eventStore.tryGet(first, bufferSize);
    System.out.println("first get size : " + entrys1.getEvents().size());
    eventStore.rollback();
    entrys1 = eventStore.tryGet(first, bufferSize);
    System.out.println("after rollback get size : " + entrys1.getEvents().size());
    Assert.assertTrue(entrys1.getEvents().size() == bufferSize / 2);
    // 继续造数据
    for (int i = bufferSize / 2; i < bufferSize; i++) {
        boolean result = eventStore.tryPut(buildEvent("1", 1L, 1L + i));
        sleep(100L);
        Assert.assertTrue(result);
    }
    Events<Event> entrys2 = eventStore.tryGet(entrys1.getPositionRange().getEnd(), bufferSize);
    System.out.println("second get size : " + entrys2.getEvents().size());
    eventStore.rollback();
    entrys2 = eventStore.tryGet(entrys1.getPositionRange().getEnd(), bufferSize);
    System.out.println("after rollback get size : " + entrys2.getEvents().size());
    Assert.assertTrue(entrys2.getEvents().size() == bufferSize);
    first = eventStore.getFirstPosition();
    lastest = eventStore.getLatestPosition();
    List<Event> entrys = new ArrayList<Event>(entrys2.getEvents());
    Assert.assertTrue(entrys.size() == bufferSize);
    Assert.assertEquals(first, entrys2.getPositionRange().getStart());
    Assert.assertEquals(lastest, entrys2.getPositionRange().getEnd());
    Assert.assertEquals(first, CanalEventUtils.createPosition(entrys.get(0)));
    Assert.assertEquals(lastest, CanalEventUtils.createPosition(entrys.get(bufferSize - 1)));
    eventStore.stop();
}

Example 15 with Position

use of com.alibaba.otter.canal.protocol.position.Position in project canal by alibaba.

the class FileMixedMetaManager method start.

public void start() {
    super.start();
    Assert.notNull(dataDir);
    if (!dataDir.exists()) {
        try {
            FileUtils.forceMkdir(dataDir);
        } catch (IOException e) {
            throw new CanalMetaManagerException(e);
        }
    }
    if (!dataDir.canRead() || !dataDir.canWrite()) {
        throw new CanalMetaManagerException("dir[" + dataDir.getPath() + "] can not read/write");
    }
    dataFileCaches = MigrateMap.makeComputingMap(new Function<String, File>() {

        public File apply(String destination) {
            return getDataFile(destination);
        }
    });
    executor = Executors.newScheduledThreadPool(1);
    destinations = MigrateMap.makeComputingMap(new Function<String, List<ClientIdentity>>() {

        public List<ClientIdentity> apply(String destination) {
            return loadClientIdentity(destination);
        }
    });
    cursors = MigrateMap.makeComputingMap(new Function<ClientIdentity, Position>() {

        public Position apply(ClientIdentity clientIdentity) {
            Position position = loadCursor(clientIdentity.getDestination(), clientIdentity);
            if (position == null) {
                // 返回一个空对象标识，避免出现异常
                return nullCursor;
            } else {
                return position;
            }
        }
    });
    updateCursorTasks = Collections.synchronizedSet(new HashSet<ClientIdentity>());
    // 启动定时工作任务
    executor.scheduleAtFixedRate(new Runnable() {

        public void run() {
            List<ClientIdentity> tasks = new ArrayList<ClientIdentity>(updateCursorTasks);
            for (ClientIdentity clientIdentity : tasks) {
                MDC.put("destination", String.valueOf(clientIdentity.getDestination()));
                try {
                    // 定时将内存中的最新值刷到file中，多次变更只刷一次
                    if (logger.isInfoEnabled()) {
                        LogPosition cursor = (LogPosition) getCursor(clientIdentity);
                        logger.info("clientId:{} cursor:[{},{},{}] address[{}]", new Object[] { clientIdentity.getClientId(), cursor.getPostion().getJournalName(), cursor.getPostion().getPosition(), cursor.getPostion().getTimestamp(), cursor.getIdentity().getSourceAddress().toString() });
                    }
                    flushDataToFile(clientIdentity.getDestination());
                    updateCursorTasks.remove(clientIdentity);
                } catch (Throwable e) {
                    // ignore
                    logger.error("period update" + clientIdentity.toString() + " curosr failed!", e);
                }
            }
        }
    }, period, period, TimeUnit.MILLISECONDS);
}