Examples with TiColumnVector com.pingcap.tikv.columnar.TiColumnVector used on opensource projects

Example 1 with TiColumnVector

use of com.pingcap.tikv.columnar.TiColumnVector in project plugins by qlangtech.

the class TiKVDataSourceDumper method startDump.

@Override
public Iterator<Map<String, Object>> startDump() {
    this.tiSession = dsFactory.getTiSession();
    // Catalog cat = this.tiSession.getCatalog();
    // TiDBInfo db = cat.getDatabase(dbName);
    // TiTableInfo tiTable = cat.getTable(db, table.getTableName());
    TiDAGRequest dagRequest = dsFactory.getTiDAGRequest(this.targetCols, tiSession, tab.tableInfo);
    Snapshot snapshot = tiSession.createSnapshot(dagRequest.getStartTs());
    // 取得的是列向量
    Iterator<TiChunk> tiChunkIterator = snapshot.tableReadChunk(dagRequest, this.partition.tasks, 1024);
    return new Iterator<Map<String, Object>>() {

        TiChunk next = null;

        int numOfRows = -1;

        int rowIndex = -1;

        TiColumnVector column = null;

        ColumnMetaData columnMetaData;

        @Override
        public boolean hasNext() {
            if (next != null) {
                if (rowIndex++ < (numOfRows - 1)) {
                    return true;
                }
                next = null;
                numOfRows = -1;
                rowIndex = -1;
            }
            boolean hasNext = tiChunkIterator.hasNext();
            if (hasNext) {
                next = tiChunkIterator.next();
                if (next == null) {
                    throw new IllegalStateException("next TiChunk can not be null");
                }
                rowIndex = 0;
                numOfRows = next.numOfRows();
            }
            return hasNext;
        }

        @Override
        public Map<String, Object> next() {
            Map<String, Object> row = new HashMap<>();
            MySQLType colType = null;
            for (int i = 0; i < targetCols.size(); i++) {
                column = next.column(i);
                if (column.isNullAt(rowIndex)) {
                    continue;
                }
                colType = column.dataType().getType();
                columnMetaData = targetCols.get(i);
                if (colType == MySQLType.TypeVarchar || colType == MySQLType.TypeString || colType == MySQLType.TypeBlob) {
                    row.put(columnMetaData.getKey(), filter(column.getUTF8String(rowIndex)));
                } else if (colType == MySQLType.TypeDate || colType == MySQLType.TypeNewDate) {
                    // FIXME 日期格式化 一个1970年的一个偏移量，按照实际情况估计要重新format一下
                    // https://cwiki.apache.org/confluence/display/Hive/LanguageManual+Types#LanguageManualTypes-date
                    row.put(columnMetaData.getKey(), // :
                    column.getLong(rowIndex));
                } else if (colType == MySQLType.TypeTimestamp || colType == MySQLType.TypeDatetime) {
                    row.put(columnMetaData.getKey(), // :
                    column.getLong(rowIndex));
                } else {
                    row.put(columnMetaData.getKey(), column.getUTF8String(rowIndex));
                }
            }
            return row;
        }
    };
}