Examples with ColumnMetaData com.qlangtech.tis.plugin.ds.ColumnMetaData used on opensource projects

Example 1 with ColumnMetaData

use of com.qlangtech.tis.plugin.ds.ColumnMetaData in project plugins by qlangtech.

the class TiKVDataSourceDumper method startDump.

@Override
public Iterator<Map<String, Object>> startDump() {
    this.tiSession = dsFactory.getTiSession();
    // Catalog cat = this.tiSession.getCatalog();
    // TiDBInfo db = cat.getDatabase(dbName);
    // TiTableInfo tiTable = cat.getTable(db, table.getTableName());
    TiDAGRequest dagRequest = dsFactory.getTiDAGRequest(this.targetCols, tiSession, tab.tableInfo);
    Snapshot snapshot = tiSession.createSnapshot(dagRequest.getStartTs());
    // 取得的是列向量
    Iterator<TiChunk> tiChunkIterator = snapshot.tableReadChunk(dagRequest, this.partition.tasks, 1024);
    return new Iterator<Map<String, Object>>() {

        TiChunk next = null;

        int numOfRows = -1;

        int rowIndex = -1;

        TiColumnVector column = null;

        ColumnMetaData columnMetaData;

        @Override
        public boolean hasNext() {
            if (next != null) {
                if (rowIndex++ < (numOfRows - 1)) {
                    return true;
                }
                next = null;
                numOfRows = -1;
                rowIndex = -1;
            }
            boolean hasNext = tiChunkIterator.hasNext();
            if (hasNext) {
                next = tiChunkIterator.next();
                if (next == null) {
                    throw new IllegalStateException("next TiChunk can not be null");
                }
                rowIndex = 0;
                numOfRows = next.numOfRows();
            }
            return hasNext;
        }

        @Override
        public Map<String, Object> next() {
            Map<String, Object> row = new HashMap<>();
            MySQLType colType = null;
            for (int i = 0; i < targetCols.size(); i++) {
                column = next.column(i);
                if (column.isNullAt(rowIndex)) {
                    continue;
                }
                colType = column.dataType().getType();
                columnMetaData = targetCols.get(i);
                if (colType == MySQLType.TypeVarchar || colType == MySQLType.TypeString || colType == MySQLType.TypeBlob) {
                    row.put(columnMetaData.getKey(), filter(column.getUTF8String(rowIndex)));
                } else if (colType == MySQLType.TypeDate || colType == MySQLType.TypeNewDate) {
                    // FIXME 日期格式化 一个1970年的一个偏移量，按照实际情况估计要重新format一下
                    // https://cwiki.apache.org/confluence/display/Hive/LanguageManual+Types#LanguageManualTypes-date
                    row.put(columnMetaData.getKey(), // :
                    column.getLong(rowIndex));
                } else if (colType == MySQLType.TypeTimestamp || colType == MySQLType.TypeDatetime) {
                    row.put(columnMetaData.getKey(), // :
                    column.getLong(rowIndex));
                } else {
                    row.put(columnMetaData.getKey(), column.getUTF8String(rowIndex));
                }
            }
            return row;
        }
    };
}

Example 2 with ColumnMetaData

use of com.qlangtech.tis.plugin.ds.ColumnMetaData in project plugins by qlangtech.

the class TestTiKVDataSourceFactory method testColMetaParse.

public void testColMetaParse() {
    GetColsMeta getColsMeta = new GetColsMeta().invoke();
    List<ColumnMetaData> table1Cols = getColsMeta.getColsMeta("table1");
    DataType type = null;
    for (ColumnMetaData col : table1Cols) {
        type = col.getType();
        System.out.println(col.getKey() + ",getDecimalDigits:" + type.getDecimalDigits() + ",columnSize:" + type.columnSize + ",type:" + type.type);
    }
}

Example 3 with ColumnMetaData

use of com.qlangtech.tis.plugin.ds.ColumnMetaData in project plugins by qlangtech.

the class TestTiKVDataSourceFactory method validateColumnMeta.

private void validateColumnMeta(boolean datetimeFormat) {
    GetColsMeta getColsMeta = new GetColsMeta().invoke(datetimeFormat);
    List<ColumnMetaData> employeesCols = getColsMeta.getEmployeesCols();
    assertNotNull(employeesCols);
    assertEquals(6, employeesCols.size());
    ColumnMetaData pk = null;
    Map<String, Integer> colTypes = Maps.newHashMap();
    colTypes.put(COL_EMP_NO, Types.BIGINT);
    colTypes.put(COL_BIRTH_DATE, datetimeFormat ? Types.DATE : Types.INTEGER);
    colTypes.put(COL_FIRST_NAME, Types.VARCHAR);
    colTypes.put(COL_LAST_NAME, Types.VARCHAR);
    colTypes.put(COL_GENDER, Types.VARCHAR);
    colTypes.put(COL_HIRE_DATE, datetimeFormat ? Types.DATE : Types.INTEGER);
    Integer colType = null;
    for (ColumnMetaData cmeta : employeesCols) {
        if (cmeta.isPk()) {
            pk = cmeta;
        }
        colType = colTypes.get(cmeta.getKey());
        assertNotNull(colType);
        assertEquals(cmeta.getKey(), (int) colType, cmeta.getType());
        System.out.println(cmeta.getIndex() + ":" + cmeta.getKey() + ":" + cmeta.getType());
    }
    assertNotNull(pk);
    assertEquals("emp_no", pk.getKey());
}

Example 4 with ColumnMetaData

use of com.qlangtech.tis.plugin.ds.ColumnMetaData in project plugins by qlangtech.

the class TestTiKVDataSourceFactory method testGetPlugin.

public void testGetPlugin() {
    GetColsMeta getColsMeta = new GetColsMeta().invoke();
    TiKVDataSourceFactory dataSourceFactory = getColsMeta.getDataSourceFactory();
    List<ColumnMetaData> employeesCols = getColsMeta.getEmployeesCols();
    TISTable dumpTable = new TISTable();
    dumpTable.setDbName(DB_NAME);
    dumpTable.setTableName(TABLE_NAME);
    dumpTable.setReflectCols(employeesCols);
    DataDumpers dataDumpers = dataSourceFactory.getDataDumpers(dumpTable);
    assertEquals(1, dataDumpers.splitCount);
    Iterator<IDataSourceDumper> dumpers = dataDumpers.dumpers;
    Map<String, Object> row = null;
    StringBuffer rowContent = null;
    int rowCount = 0;
    while (dumpers.hasNext()) {
        IDataSourceDumper dumper = dumpers.next();
        // assertEquals(300024, );
        assertTrue(dumper.getRowSize() > 0);
        try {
            Iterator<Map<String, Object>> rowIterator = dumper.startDump();
            while (rowIterator.hasNext()) {
                rowContent = new StringBuffer();
                row = rowIterator.next();
                // }
                for (Map.Entry<String, Object> entry : row.entrySet()) {
                    rowContent.append(entry.getKey()).append(":").append(entry.getValue()).append(",");
                }
                rowCount++;
                System.out.println(rowContent);
            }
        } finally {
            dumper.closeResource();
        }
    }
    assertEquals(300024, rowCount);
}

Example 5 with ColumnMetaData

use of com.qlangtech.tis.plugin.ds.ColumnMetaData in project plugins by qlangtech.

the class TestPGDataSourceFactory method testShowTables.

public void testShowTables() {
    String createSQL = IOUtils.loadResourceFromClasspath(TestPGDataSourceFactory.class, "create-sql-instancedetail.sql");
    // 之后数据库如果没有了可以用下面的这个SQL 再把数据库给跑起来的
    System.out.println(createSQL);
    PGDataSourceFactory dsFactory = new PGDataSourceFactory();
    dsFactory.userName = "postgres";
    dsFactory.password = "123456";
    dsFactory.dbName = "tis";
    dsFactory.port = 5432;
    dsFactory.nodeDesc = "192.168.28.201";
    dsFactory.tabSchema = "public";
    String instancedetail = "instancedetail";
    List<String> tables = dsFactory.getTablesInDB();
    assertEquals(1, tables.size());
    tables.contains(instancedetail);
    List<ColumnMetaData> tableMetadata = dsFactory.getTableMetadata(instancedetail);
    assertTrue(tableMetadata.size() > 0);
    for (ColumnMetaData col : tableMetadata) {
        System.out.println(col.getKey() + "  " + col.getType());
    }
}