Examples with Header com.alibaba.otter.canal.protocol.CanalEntry.Header used on opensource projects

Example 1 with Header

use of com.alibaba.otter.canal.protocol.CanalEntry.Header in project canal by alibaba.

the class EventTransactionBufferTest method testForceFlush.

@Test
public void testForceFlush() {
    final int bufferSize = 64;
    EventTransactionBuffer buffer = new EventTransactionBuffer();
    buffer.setBufferSize(bufferSize);
    buffer.setFlushCallback(transaction -> {
        Assert.assertEquals(bufferSize, transaction.size());
        System.out.println("\n>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>");
        for (Entry data : transaction) {
            Header header = data.getHeader();
            Date date = new Date(header.getExecuteTime());
            SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT);
            if (data.getEntryType() == EntryType.TRANSACTIONBEGIN || data.getEntryType() == EntryType.TRANSACTIONEND) {
                // System.out.println(MessageFormat.format(messgae, new
                // Object[] {
                // Thread.currentThread().getName(),
                // header.getLogfilename(), header.getLogfileoffset(),
                // format.format(date),
                // data.getEntry().getEntryType(), "" }));
                System.out.println(data.getEntryType());
            } else {
                System.out.println(MessageFormat.format(messgae, new Object[] { Thread.currentThread().getName(), header.getLogfileName(), header.getLogfileOffset(), format.format(date), header.getSchemaName(), header.getTableName() }));
            }
        }
        System.out.println("<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<\n");
    });
    buffer.start();
    try {
        for (int i = 0; i < bufferSize * 2 + 1; i++) {
            buffer.add(buildEntry("1", 1L + i, 40L + i));
        }
    } catch (InterruptedException e) {
        Assert.fail(e.getMessage());
    }
    buffer.stop();
}

Example 2 with Header

use of com.alibaba.otter.canal.protocol.CanalEntry.Header in project canal by alibaba.

the class LogEventConvert method parseMariaGTIDLogEvent.

private Entry parseMariaGTIDLogEvent(LogEvent logEvent) {
    LogHeader logHeader = logEvent.getHeader();
    Pair.Builder builder = Pair.newBuilder();
    builder.setKey("gtid");
    if (logEvent instanceof MariaGtidLogEvent) {
        builder.setValue(((MariaGtidLogEvent) logEvent).getGtidStr());
    } else if (logEvent instanceof MariaGtidListLogEvent) {
        builder.setValue(((MariaGtidListLogEvent) logEvent).getGtidStr());
    }
    Header header = createHeader(logHeader, "", "", EventType.GTID);
    return createEntry(header, EntryType.GTIDLOG, builder.build().toByteString());
}

Example 3 with Header

use of com.alibaba.otter.canal.protocol.CanalEntry.Header in project canal by alibaba.

the class LogEventConvert method createHeader.

private Header createHeader(LogHeader logHeader, String schemaName, String tableName, EventType eventType, Integer rowsCount) {
    // header会做信息冗余,方便以后做检索或者过滤
    Header.Builder headerBuilder = Header.newBuilder();
    headerBuilder.setVersion(version);
    headerBuilder.setLogfileName(logHeader.getLogFileName());
    // 记录的是该binlog的start offest
    headerBuilder.setLogfileOffset(logHeader.getLogPos() - logHeader.getEventLen());
    headerBuilder.setServerId(logHeader.getServerId());
    // 经过java输出后所有的编码为unicode
    headerBuilder.setServerenCode(UTF_8);
    headerBuilder.setExecuteTime(logHeader.getWhen() * 1000L);
    headerBuilder.setSourceType(Type.MYSQL);
    if (eventType != null) {
        headerBuilder.setEventType(eventType);
    }
    if (schemaName != null) {
        headerBuilder.setSchemaName(schemaName);
    }
    if (tableName != null) {
        headerBuilder.setTableName(tableName);
    }
    headerBuilder.setEventLength(logHeader.getEventLen());
    // enable gtid position
    if (StringUtils.isNotEmpty(logHeader.getGtidSetStr())) {
        headerBuilder.setGtid(logHeader.getGtidSetStr());
    }
    // add current gtid
    if (StringUtils.isNotEmpty(logHeader.getCurrentGtid())) {
        Pair pair = createSpecialPair("curtGtid", logHeader.getCurrentGtid());
        headerBuilder.addProps(pair);
    }
    // add current gtid sequence no
    if (StringUtils.isNotEmpty(logHeader.getCurrentGtidSn())) {
        Pair pair = createSpecialPair("curtGtidSn", logHeader.getCurrentGtidSn());
        headerBuilder.addProps(pair);
    }
    // add current gtid last committed
    if (StringUtils.isNotEmpty(logHeader.getCurrentGtidLastCommit())) {
        Pair pair = createSpecialPair("curtGtidLct", logHeader.getCurrentGtidLastCommit());
        headerBuilder.addProps(pair);
    }
    // add rowsCount suppport
    if (rowsCount > 0) {
        Pair pair = createSpecialPair("rowsCount", String.valueOf(rowsCount));
        headerBuilder.addProps(pair);
    }
    return headerBuilder.build();
}

Example 4 with Header

use of com.alibaba.otter.canal.protocol.CanalEntry.Header in project canal by alibaba.

the class LogEventConvert method buildQueryEntry.

private Entry buildQueryEntry(String queryString, LogHeader logHeader) {
    Header header = createHeader(logHeader, "", "", EventType.QUERY);
    RowChange.Builder rowChangeBuider = RowChange.newBuilder();
    rowChangeBuider.setSql(queryString);
    rowChangeBuider.setEventType(EventType.QUERY);
    return createEntry(header, EntryType.ROWDATA, rowChangeBuider.build().toByteString());
}

Example 5 with Header

use of com.alibaba.otter.canal.protocol.CanalEntry.Header in project canal by alibaba.

the class LogEventConvert method parseRowsEvent.

private Entry parseRowsEvent(RowsLogEvent event) {
    if (filterRows) {
        return null;
    }
    try {
        TableMapLogEvent table = event.getTable();
        if (table == null) {
            // tableId对应的记录不存在
            throw new TableIdNotFoundException("not found tableId:" + event.getTableId());
        }
        String fullname = table.getDbName() + "." + table.getTableName();
        // check name filter
        if (nameFilter != null && !nameFilter.filter(fullname)) {
            return null;
        }
        if (nameBlackFilter != null && nameBlackFilter.filter(fullname)) {
            return null;
        }
        if (tableMetaCache.isOnRDS() && "mysql.ha_health_check".equals(fullname)) {
            // 忽略rds模式的mysql.ha_health_check心跳数据
            return null;
        }
        EventType eventType = null;
        int type = event.getHeader().getType();
        if (LogEvent.WRITE_ROWS_EVENT_V1 == type || LogEvent.WRITE_ROWS_EVENT == type) {
            eventType = EventType.INSERT;
        } else if (LogEvent.UPDATE_ROWS_EVENT_V1 == type || LogEvent.UPDATE_ROWS_EVENT == type) {
            eventType = EventType.UPDATE;
        } else if (LogEvent.DELETE_ROWS_EVENT_V1 == type || LogEvent.DELETE_ROWS_EVENT == type) {
            eventType = EventType.DELETE;
        } else {
            throw new CanalParseException("unsupport event type :" + event.getHeader().getType());
        }
        Header header = createHeader(binlogFileName, event.getHeader(), table.getDbName(), table.getTableName(), eventType);
        RowChange.Builder rowChangeBuider = RowChange.newBuilder();
        rowChangeBuider.setTableId(event.getTableId());
        rowChangeBuider.setIsDdl(false);
        rowChangeBuider.setEventType(eventType);
        RowsLogBuffer buffer = event.getRowsBuf(charset.name());
        BitSet columns = event.getColumns();
        BitSet changeColumns = event.getChangeColumns();
        boolean tableError = false;
        TableMeta tableMeta = null;
        if (tableMetaCache != null) {
            // 入错存在table meta cache
            tableMeta = getTableMeta(table.getDbName(), table.getTableName(), true);
            if (tableMeta == null) {
                tableError = true;
                if (!filterTableError) {
                    throw new CanalParseException("not found [" + fullname + "] in db , pls check!");
                }
            }
        }
        while (buffer.nextOneRow(columns)) {
            // 处理row记录
            RowData.Builder rowDataBuilder = RowData.newBuilder();
            if (EventType.INSERT == eventType) {
                // insert的记录放在before字段中
                tableError |= parseOneRow(rowDataBuilder, event, buffer, columns, true, tableMeta);
            } else if (EventType.DELETE == eventType) {
                // delete的记录放在before字段中
                tableError |= parseOneRow(rowDataBuilder, event, buffer, columns, false, tableMeta);
            } else {
                // update需要处理before/after
                tableError |= parseOneRow(rowDataBuilder, event, buffer, columns, false, tableMeta);
                if (!buffer.nextOneRow(changeColumns)) {
                    rowChangeBuider.addRowDatas(rowDataBuilder.build());
                    break;
                }
                tableError |= parseOneRow(rowDataBuilder, event, buffer, changeColumns, true, tableMeta);
            }
            rowChangeBuider.addRowDatas(rowDataBuilder.build());
        }
        RowChange rowChange = rowChangeBuider.build();
        if (tableError) {
            Entry entry = createEntry(header, EntryType.ROWDATA, ByteString.EMPTY);
            logger.warn("table parser error : {}storeValue: {}", entry.toString(), rowChange.toString());
            return null;
        } else {
            Entry entry = createEntry(header, EntryType.ROWDATA, rowChangeBuider.build().toByteString());
            return entry;
        }
    } catch (Exception e) {
        throw new CanalParseException("parse row data failed.", e);
    }
}