Examples with DataDumpers com.qlangtech.tis.plugin.ds.DataDumpers used on opensource projects

Example 1 with DataDumpers

use of com.qlangtech.tis.plugin.ds.DataDumpers in project plugins by qlangtech.

the class TestTiKVDataSourceFactory method testGetPlugin.

public void testGetPlugin() {
    GetColsMeta getColsMeta = new GetColsMeta().invoke();
    TiKVDataSourceFactory dataSourceFactory = getColsMeta.getDataSourceFactory();
    List<ColumnMetaData> employeesCols = getColsMeta.getEmployeesCols();
    TISTable dumpTable = new TISTable();
    dumpTable.setDbName(DB_NAME);
    dumpTable.setTableName(TABLE_NAME);
    dumpTable.setReflectCols(employeesCols);
    DataDumpers dataDumpers = dataSourceFactory.getDataDumpers(dumpTable);
    assertEquals(1, dataDumpers.splitCount);
    Iterator<IDataSourceDumper> dumpers = dataDumpers.dumpers;
    Map<String, Object> row = null;
    StringBuffer rowContent = null;
    int rowCount = 0;
    while (dumpers.hasNext()) {
        IDataSourceDumper dumper = dumpers.next();
        // assertEquals(300024, );
        assertTrue(dumper.getRowSize() > 0);
        try {
            Iterator<Map<String, Object>> rowIterator = dumper.startDump();
            while (rowIterator.hasNext()) {
                rowContent = new StringBuffer();
                row = rowIterator.next();
                // }
                for (Map.Entry<String, Object> entry : row.entrySet()) {
                    rowContent.append(entry.getKey()).append(":").append(entry.getValue()).append(",");
                }
                rowCount++;
                System.out.println(rowContent);
            }
        } finally {
            dumper.closeResource();
        }
    }
    assertEquals(300024, rowCount);
}

Example 2 with DataDumpers

use of com.qlangtech.tis.plugin.ds.DataDumpers in project plugins by qlangtech.

the class TestDorisSourceFactory method testDataDumpers.

public void testDataDumpers() throws Exception {
    DorisSourceFactory dataSourceFactory = getDorisSourceFactory();
    TISTable dumpTable = new TISTable();
    dumpTable.setSelectSql("SELECT emp_no,birth_date,first_name,last_name,gender,hire_date FROM employees");
    dumpTable.setTableName("employees");
    DataDumpers dataDumpers = dataSourceFactory.getDataDumpers(dumpTable);
    assertNotNull("dataDumpers can not be null", dataDumpers);
    assertEquals(1, dataDumpers.splitCount);
    Iterator<IDataSourceDumper> dumpers = dataDumpers.dumpers;
    Map<String, String> row = null;
    assertTrue("must contain a dumper", dumpers.hasNext());
    IDataSourceDumper dumper = dumpers.next();
    assertNotNull(dumper);
    assertEquals("jdbc:mysql://192.168.28.202:9030/", dumper.getDbHost());
// dumper.closeResource();
}

Example 3 with DataDumpers

use of com.qlangtech.tis.plugin.ds.DataDumpers in project plugins by qlangtech.

the class TisDataXTiDBReader method getTiKVDataSource.

private static final List<IDataSourceDumper> getTiKVDataSource(Configuration config, Optional<Long> regionId) {
    List<String> cols = config.getList("column", String.class);
    String tableName = config.getString("table");
    Configuration connection = config.getConfiguration("connection");
    TiKVDataSourceFactory sourceFactory = new TiKVDataSourceFactory();
    sourceFactory.pdAddrs = connection.getString("[0].pdAddrs");
    sourceFactory.dbName = connection.getString("[0].dbName");
    // sourceFactory.datetimeFormat = connection.getBool("datetimeFormat");
    if (StringUtils.isBlank(sourceFactory.pdAddrs) || StringUtils.isBlank(sourceFactory.dbName)) {
        throw new IllegalStateException("param 'pdAddrs' or 'dbName' can not be null,connection:" + connection.toJSON());
    }
    List<ColumnMetaData> tableMetadata = sourceFactory.getTableMetadata(tableName);
    TISTable table = new TISTable();
    table.setTableName(tableName);
    table.setReflectCols(tableMetadata.stream().filter((cmeta) -> cols.contains(cmeta.getKey())).collect(Collectors.toList()));
    DataDumpers dataDumpers = sourceFactory.getDataDumpers(table, regionId);
    List<IDataSourceDumper> dumpers = Lists.newArrayList(dataDumpers.dumpers);
    return dumpers;
}