Examples with ColumnParser org.apache.rocketmq.connect.jdbc.schema.column.ColumnParser used on opensource projects

Example 1 with ColumnParser

use of org.apache.rocketmq.connect.jdbc.schema.column.ColumnParser in project rocketmq-externals by apache.

the class Querier method poll.

public void poll() {
    try {
        PreparedStatement stmt;
        LinkedList<Table> tableLinkedList = new LinkedList<>();
        for (Map.Entry<String, Database> entry : schema.getDbMap().entrySet()) {
            String db = entry.getKey();
            Iterator<Map.Entry<String, Table>> iterator = entry.getValue().getTableMap().entrySet().iterator();
            while (iterator.hasNext()) {
                StringBuilder query = new StringBuilder("select * from ");
                Map.Entry<String, Table> tableEntry = iterator.next();
                String tb = tableEntry.getKey();
                query.append(db + "." + tb);
                Table t = tableEntry.getValue();
                Map<String, String> tableFilterMap = t.getFilterMap();
                if (tableFilterMap != null && !tableFilterMap.keySet().contains("NO-FILTER")) {
                    query = query.append(" where ");
                    int count = 0;
                    for (String key : tableFilterMap.keySet()) {
                        count++;
                        String value = tableFilterMap.get(key);
                        if (count != 1) {
                            query.append(" and ");
                        }
                        String condition = key + "=" + "'" + value + "'";
                        query.append(condition);
                    }
                }
                stmt = connection.prepareStatement(query.toString());
                ResultSet rs;
                rs = stmt.executeQuery();
                List<String> colList = tableEntry.getValue().getColList();
                List<String> dataTypeList = tableEntry.getValue().getRawDataTypeList();
                List<ColumnParser> parserList = tableEntry.getValue().getParserList();
                while (rs.next()) {
                    Table table = new Table(db, tb);
                    // System.out.print("|");
                    table.setColList(colList);
                    table.setRawDataTypeList(dataTypeList);
                    table.setParserList(parserList);
                    for (String string : colList) {
                        table.getDataList().add(rs.getObject(string));
                    // System.out.print(string + " : " + rs.getObject(string) + "|");
                    }
                    tableLinkedList.add(table);
                }
                rs.close();
                stmt.close();
            }
        }
        list = tableLinkedList;
    } catch (SQLException e) {
        log.error("fail to poll data, {}", e);
    }
}

Example 2 with ColumnParser

use of org.apache.rocketmq.connect.jdbc.schema.column.ColumnParser in project rocketmq-externals by apache.

the class TimestampIncrementingQuerier method poll.

public void poll() {
    try {
        list.clear();
        Connection conn = dataSource.getConnection();
        for (Map.Entry<String, Database> entry : schema.getDbMap().entrySet()) {
            String db = entry.getKey();
            log.info("{} database is loading", db);
            Iterator<Map.Entry<String, Table>> iterator = entry.getValue().getTableMap().entrySet().iterator();
            while (iterator.hasNext()) {
                Map.Entry<String, Table> tableEntry = iterator.next();
                String tb = tableEntry.getKey();
                log.info("{} table is loading", tb);
                name = db + "." + tb;
                storeRecord(name);
                createPreparedStatement(conn);
                ResultSet rs;
                rs = executeQuery();
                List<String> colList = tableEntry.getValue().getColList();
                List<String> DataTypeList = tableEntry.getValue().getRawDataTypeList();
                List<ColumnParser> ParserList = tableEntry.getValue().getParserList();
                while (rs.next()) {
                    Table table = new Table(db, tb);
                    System.out.print("|");
                    table.setColList(colList);
                    table.setRawDataTypeList(DataTypeList);
                    table.setParserList(ParserList);
                    for (String string : colList) {
                        table.getDataList().add(rs.getObject(string));
                        System.out.print(string + " : " + rs.getObject(string) + "|");
                    }
                    incrementingOffset = incrementingOffset > rs.getInt(incrementingColumn) ? incrementingOffset : rs.getInt(incrementingColumn);
                    timestampOffset = timestampOffset > rs.getTimestamp(timestampColumn).getTime() ? timestampOffset : rs.getTimestamp(timestampColumn).getTime();
                    System.out.println(timestampOffset);
                    list.add(table);
                    System.out.println();
                }
                extractRecord(name);
                incrementingOffset = 0L;
                timestampOffset = 0L;
            }
        }
        conn.close();
    } catch (SQLException e) {
        e.printStackTrace();
    }
}

Example 3 with ColumnParser

use of org.apache.rocketmq.connect.jdbc.schema.column.ColumnParser in project rocketmq-externals by apache.

the class Database method init.

public void init() throws SQLException {
    PreparedStatement ps = null;
    ResultSet rs = null;
    try {
        ps = connection.prepareStatement(SQL);
        ps.setString(1, name);
        rs = ps.executeQuery();
        while (rs.next()) {
            String tableName = rs.getString(1);
            String colName = rs.getString(2);
            String dataType = rs.getString(3);
            String colType = rs.getString(4);
            String charset = rs.getString(5);
            ColumnParser columnParser = ColumnParser.getColumnParser(dataType, colType, charset);
            if (!tableWhiteList.contains(tableName)) {
                continue;
            }
            if (!tableMap.containsKey(tableName)) {
                addTable(tableName);
            }
            Table table = tableMap.get(tableName);
            table.addCol(colName);
            table.addParser(columnParser);
            table.addRawDataType(dataType);
            table.setFilterMap(tableFilterMap.get(tableName));
        }
    } finally {
        if (rs != null) {
            rs.close();
        }
        if (ps != null) {
            ps.close();
        }
    }
}