Examples with MetaSnapshotDO com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaSnapshotDO used on opensource projects

Example 1 with MetaSnapshotDO

use of com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaSnapshotDO in project canal by alibaba.

the class DatabaseTableMeta method applySnapshotToDB.

/**
 * 发布数据到console上
 */
private boolean applySnapshotToDB(EntryPosition position, boolean init) {
    // 获取一份快照
    Map<String, String> schemaDdls = null;
    lock.readLock().lock();
    try {
        if (!init && !hasNewDdl) {
            // 如果是持续构建,则识别一下是否有DDL变更过,如果没有就忽略了
            return false;
        }
        this.hasNewDdl = false;
        schemaDdls = memoryTableMeta.snapshot();
    } finally {
        lock.readLock().unlock();
    }
    MemoryTableMeta tmpMemoryTableMeta = new MemoryTableMeta();
    for (Map.Entry<String, String> entry : schemaDdls.entrySet()) {
        tmpMemoryTableMeta.apply(position, entry.getKey(), entry.getValue(), null);
    }
    // 基于临时内存对象进行对比
    boolean compareAll = true;
    for (Schema schema : tmpMemoryTableMeta.getRepository().getSchemas()) {
        for (String table : schema.showTables()) {
            String fullName = schema.getName() + "." + table;
            if (blackFilter == null || !blackFilter.filter(fullName)) {
                if (filter == null || filter.filter(fullName)) {
                    // 在生成snapshot时重新过滤一遍
                    if (!compareTableMetaDbAndMemory(connection, tmpMemoryTableMeta, schema.getName(), table)) {
                        compareAll = false;
                    }
                }
            }
        }
    }
    if (compareAll) {
        Map<String, String> content = new HashMap<>();
        content.put("destination", destination);
        content.put("binlogFile", position.getJournalName());
        content.put("binlogOffest", String.valueOf(position.getPosition()));
        content.put("binlogMasterId", String.valueOf(position.getServerId()));
        content.put("binlogTimestamp", String.valueOf(position.getTimestamp()));
        content.put("data", JSON.toJSONString(schemaDdls));
        if (content.isEmpty()) {
            throw new RuntimeException("apply failed caused by content is empty in applySnapshotToDB");
        }
        MetaSnapshotDO snapshotDO = new MetaSnapshotDO();
        try {
            BeanUtils.populate(snapshotDO, content);
            metaSnapshotDAO.insert(snapshotDO);
        } catch (Throwable e) {
            if (isUkDuplicateException(e)) {
                // 忽略掉重复的位点
                logger.info("dup apply snapshot use position : " + position + " , just ignore");
            } else {
                throw new CanalParseException("apply failed caused by : " + e.getMessage(), e);
            }
        }
        return true;
    } else {
        logger.error("compare failed , check log");
    }
    return false;
}

Example 2 with MetaSnapshotDO

use of com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaSnapshotDO in project canal by alibaba.

the class DatabaseTableMeta method buildMemFromSnapshot.

private EntryPosition buildMemFromSnapshot(EntryPosition position) {
    try {
        MetaSnapshotDO snapshotDO = metaSnapshotDAO.findByTimestamp(destination, position.getTimestamp());
        if (snapshotDO == null) {
            return null;
        }
        String binlogFile = snapshotDO.getBinlogFile();
        Long binlogOffest = snapshotDO.getBinlogOffest();
        String binlogMasterId = snapshotDO.getBinlogMasterId();
        Long binlogTimestamp = snapshotDO.getBinlogTimestamp();
        EntryPosition snapshotPosition = new EntryPosition(binlogFile, binlogOffest == null ? 0l : binlogOffest, binlogTimestamp == null ? 0l : binlogTimestamp, Long.valueOf(binlogMasterId == null ? "-2" : binlogMasterId));
        // data存储为Map<String,String>，每个分库一套建表
        String sqlData = snapshotDO.getData();
        JSONObject jsonObj = JSON.parseObject(sqlData);
        for (Map.Entry entry : jsonObj.entrySet()) {
            // 记录到内存
            if (!memoryTableMeta.apply(snapshotPosition, ObjectUtils.toString(entry.getKey()), ObjectUtils.toString(entry.getValue()), null)) {
                return null;
            }
        }
        return snapshotPosition;
    } catch (Throwable e) {
        throw new CanalParseException("apply failed caused by : " + e.getMessage(), e);
    }
}

Example 3 with MetaSnapshotDO

use of com.alibaba.otter.canal.parse.inbound.mysql.tsdb.dao.MetaSnapshotDO in project canal by alibaba.

the class MetaSnapshotDAOTest method testSimple.

@Ignore
@Test
public void testSimple() {
    MetaSnapshotDO metaSnapshotDO = new MetaSnapshotDO();
    metaSnapshotDO.setDestination("test");
    metaSnapshotDO.setBinlogFile("000001");
    metaSnapshotDO.setBinlogOffest(4L);
    metaSnapshotDO.setBinlogMasterId("1");
    metaSnapshotDO.setBinlogTimestamp(System.currentTimeMillis() - 7300 * 1000);
    metaSnapshotDO.setData("test");
    metaSnapshotDAO.insert(metaSnapshotDO);
    MetaSnapshotDO snapshotDO = metaSnapshotDAO.findByTimestamp("test", System.currentTimeMillis());
    System.out.println(snapshotDO);
    int count = metaSnapshotDAO.deleteByTimestamp("test", 7200);
    System.out.println(count);
}