Examples with FieldMeta com.alibaba.otter.canal.parse.inbound.TableMeta.FieldMeta used on opensource projects

Example 1 with FieldMeta

use of com.alibaba.otter.canal.parse.inbound.TableMeta.FieldMeta in project canal by alibaba.

the class TableMetaCache method parserTableMeta.

private List<FieldMeta> parserTableMeta(ResultSetPacket packet) {
    Map<String, Integer> nameMaps = new HashMap<String, Integer>(6, 1f);
    int index = 0;
    for (FieldPacket fieldPacket : packet.getFieldDescriptors()) {
        nameMaps.put(fieldPacket.getOriginalName(), index++);
    }
    int size = packet.getFieldDescriptors().size();
    int count = packet.getFieldValues().size() / packet.getFieldDescriptors().size();
    List<FieldMeta> result = new ArrayList<FieldMeta>();
    for (int i = 0; i < count; i++) {
        FieldMeta meta = new FieldMeta();
        // 做一个优化，使用String.intern()，共享String对象，减少内存使用
        meta.setColumnName(packet.getFieldValues().get(nameMaps.get(COLUMN_NAME) + i * size).intern());
        meta.setColumnType(packet.getFieldValues().get(nameMaps.get(COLUMN_TYPE) + i * size));
        meta.setIsNullable(packet.getFieldValues().get(nameMaps.get(IS_NULLABLE) + i * size));
        meta.setIskey(packet.getFieldValues().get(nameMaps.get(COLUMN_KEY) + i * size));
        meta.setDefaultValue(packet.getFieldValues().get(nameMaps.get(COLUMN_DEFAULT) + i * size));
        meta.setExtra(packet.getFieldValues().get(nameMaps.get(EXTRA) + i * size));
        result.add(meta);
    }
    return result;
}

Example 2 with FieldMeta

use of com.alibaba.otter.canal.parse.inbound.TableMeta.FieldMeta in project canal by alibaba.

the class LogEventConvert method parseOneRow.

private boolean parseOneRow(RowData.Builder rowDataBuilder, RowsLogEvent event, RowsLogBuffer buffer, BitSet cols, boolean isAfter, TableMeta tableMeta) throws UnsupportedEncodingException {
    int columnCnt = event.getTable().getColumnCnt();
    ColumnInfo[] columnInfo = event.getTable().getColumnInfo();
    boolean tableError = false;
    // check table fileds count，只能处理加字段
    boolean existRDSNoPrimaryKey = false;
    if (tableMeta != null && columnInfo.length > tableMeta.getFileds().size()) {
        if (tableMetaCache.isOnRDS()) {
            // 特殊处理下RDS的场景
            List<FieldMeta> primaryKeys = tableMeta.getPrimaryFields();
            if (primaryKeys == null || primaryKeys.isEmpty()) {
                if (columnInfo.length == tableMeta.getFileds().size() + 1 && columnInfo[columnInfo.length - 1].type == LogEvent.MYSQL_TYPE_LONGLONG) {
                    existRDSNoPrimaryKey = true;
                }
            }
        }
        if (!existRDSNoPrimaryKey) {
            // online ddl增加字段操作步骤：
            // 1. 新增一张临时表，将需要做ddl表的数据全量导入
            // 2. 在老表上建立I/U/D的trigger，增量的将数据插入到临时表
            // 3. 锁住应用请求，将临时表rename为老表的名字，完成增加字段的操作
            // 尝试做一次reload，可能因为ddl没有正确解析，或者使用了类似online ddl的操作
            // 因为online ddl没有对应表名的alter语法，所以不会有clear cache的操作
            // 强制重新获取一次
            tableMeta = getTableMeta(event.getTable().getDbName(), event.getTable().getTableName(), false);
            if (tableMeta == null) {
                tableError = true;
                if (!filterTableError) {
                    throw new CanalParseException("not found [" + event.getTable().getDbName() + "." + event.getTable().getTableName() + "] in db , pls check!");
                }
            }
            // 在做一次判断
            if (tableMeta != null && columnInfo.length > tableMeta.getFileds().size()) {
                tableError = true;
                if (!filterTableError) {
                    throw new CanalParseException("column size is not match for table:" + tableMeta.getFullName() + "," + columnInfo.length + " vs " + tableMeta.getFileds().size());
                }
            }
        } else {
            logger.warn("[" + event.getTable().getDbName() + "." + event.getTable().getTableName() + "] is no primary key , skip alibaba_rds_row_id column");
        }
    }
    for (int i = 0; i < columnCnt; i++) {
        ColumnInfo info = columnInfo[i];
        // mysql 5.6开始支持nolob/mininal类型,并不一定记录所有的列,需要进行判断
        if (!cols.get(i)) {
            continue;
        }
        if (existRDSNoPrimaryKey && i == columnCnt - 1 && info.type == LogEvent.MYSQL_TYPE_LONGLONG) {
            // 不解析最后一列
            buffer.nextValue(info.type, info.meta, false);
            continue;
        }
        Column.Builder columnBuilder = Column.newBuilder();
        FieldMeta fieldMeta = null;
        if (tableMeta != null && !tableError) {
            // 处理file meta
            fieldMeta = tableMeta.getFileds().get(i);
            columnBuilder.setName(fieldMeta.getColumnName());
            columnBuilder.setIsKey(fieldMeta.isKey());
            // 增加mysql type类型,issue 73
            columnBuilder.setMysqlType(fieldMeta.getColumnType());
        }
        columnBuilder.setIndex(i);
        columnBuilder.setIsNull(false);
        // fixed issue
        // https://github.com/alibaba/canal/issues/66，特殊处理binary/varbinary，不能做编码处理
        boolean isBinary = false;
        if (fieldMeta != null) {
            if (StringUtils.containsIgnoreCase(fieldMeta.getColumnType(), "VARBINARY")) {
                isBinary = true;
            } else if (StringUtils.containsIgnoreCase(fieldMeta.getColumnType(), "BINARY")) {
                isBinary = true;
            }
        }
        buffer.nextValue(info.type, info.meta, isBinary);
        if (existRDSNoPrimaryKey && i == columnCnt - 1 && info.type == LogEvent.MYSQL_TYPE_LONGLONG) {
            // 不解析最后一列
            continue;
        }
        int javaType = buffer.getJavaType();
        if (buffer.isNull()) {
            columnBuilder.setIsNull(true);
        } else {
            final Serializable value = buffer.getValue();
            // 处理各种类型
            switch(javaType) {
                case Types.INTEGER:
                case Types.TINYINT:
                case Types.SMALLINT:
                case Types.BIGINT:
                    // 处理unsigned类型
                    Number number = (Number) value;
                    if (fieldMeta != null && fieldMeta.isUnsigned() && number.longValue() < 0) {
                        switch(buffer.getLength()) {
                            case 1:
                                /* MYSQL_TYPE_TINY */
                                columnBuilder.setValue(String.valueOf(Integer.valueOf(TINYINT_MAX_VALUE + number.intValue())));
                                // 往上加一个量级
                                javaType = Types.SMALLINT;
                                break;
                            case 2:
                                /* MYSQL_TYPE_SHORT */
                                columnBuilder.setValue(String.valueOf(Integer.valueOf(SMALLINT_MAX_VALUE + number.intValue())));
                                // 往上加一个量级
                                javaType = Types.INTEGER;
                                break;
                            case 3:
                                /* MYSQL_TYPE_INT24 */
                                columnBuilder.setValue(String.valueOf(Integer.valueOf(MEDIUMINT_MAX_VALUE + number.intValue())));
                                // 往上加一个量级
                                javaType = Types.INTEGER;
                                break;
                            case 4:
                                /* MYSQL_TYPE_LONG */
                                columnBuilder.setValue(String.valueOf(Long.valueOf(INTEGER_MAX_VALUE + number.longValue())));
                                // 往上加一个量级
                                javaType = Types.BIGINT;
                                break;
                            case 8:
                                /* MYSQL_TYPE_LONGLONG */
                                columnBuilder.setValue(BIGINT_MAX_VALUE.add(BigInteger.valueOf(number.longValue())).toString());
                                // 往上加一个量级，避免执行出错
                                javaType = Types.DECIMAL;
                                break;
                        }
                    } else {
                        // 对象为number类型，直接valueof即可
                        columnBuilder.setValue(String.valueOf(value));
                    }
                    break;
                // float
                case Types.REAL:
                case // double
                Types.DOUBLE:
                    // 对象为number类型，直接valueof即可
                    columnBuilder.setValue(String.valueOf(value));
                    break;
                case // bit
                Types.BIT:
                    // 对象为number类型
                    columnBuilder.setValue(String.valueOf(value));
                    break;
                case Types.DECIMAL:
                    columnBuilder.setValue(((BigDecimal) value).toPlainString());
                    break;
                case Types.TIMESTAMP:
                // break;
                case Types.TIME:
                case Types.DATE:
                    // 需要处理year
                    columnBuilder.setValue(value.toString());
                    break;
                case Types.BINARY:
                case Types.VARBINARY:
                case Types.LONGVARBINARY:
                    // meta，按编码解析text
                    if (fieldMeta != null && isText(fieldMeta.getColumnType())) {
                        columnBuilder.setValue(new String((byte[]) value, charset));
                        javaType = Types.CLOB;
                    } else {
                        // byte数组，直接使用iso-8859-1保留对应编码，浪费内存
                        columnBuilder.setValue(new String((byte[]) value, ISO_8859_1));
                        javaType = Types.BLOB;
                    }
                    break;
                case Types.CHAR:
                case Types.VARCHAR:
                    columnBuilder.setValue(value.toString());
                    break;
                default:
                    columnBuilder.setValue(value.toString());
            }
        }
        columnBuilder.setSqlType(javaType);
        // 设置是否update的标记位
        columnBuilder.setUpdated(isAfter && isUpdate(rowDataBuilder.getBeforeColumnsList(), columnBuilder.getIsNull() ? null : columnBuilder.getValue(), i));
        if (isAfter) {
            rowDataBuilder.addAfterColumns(columnBuilder.build());
        } else {
            rowDataBuilder.addBeforeColumns(columnBuilder.build());
        }
    }
    return tableError;
}

Example 3 with FieldMeta

use of com.alibaba.otter.canal.parse.inbound.TableMeta.FieldMeta in project canal by alibaba.

the class TableMetaCacheTest method testSimple.

@Test
public void testSimple() {
    MysqlConnection connection = new MysqlConnection(new InetSocketAddress("127.0.0.1", 3306), "xxxxx", "xxxxx");
    try {
        connection.connect();
    } catch (IOException e) {
        Assert.fail(e.getMessage());
    }
    TableMetaCache cache = new TableMetaCache(connection);
    TableMeta meta = cache.getTableMeta("otter1", "otter_stability1");
    Assert.assertNotNull(meta);
    for (FieldMeta field : meta.getFileds()) {
        System.out.println("filed :" + field.getColumnName() + " , isKey : " + field.isKey() + " , isNull : " + field.isNullable());
    }
}