Examples with TISTable com.qlangtech.tis.plugin.ds.TISTable used on opensource projects

Example 1 with TISTable

use of com.qlangtech.tis.plugin.ds.TISTable in project plugins by qlangtech.

the class TestTiKVDataSourceFactory method testGetPlugin.

public void testGetPlugin() {
    GetColsMeta getColsMeta = new GetColsMeta().invoke();
    TiKVDataSourceFactory dataSourceFactory = getColsMeta.getDataSourceFactory();
    List<ColumnMetaData> employeesCols = getColsMeta.getEmployeesCols();
    TISTable dumpTable = new TISTable();
    dumpTable.setDbName(DB_NAME);
    dumpTable.setTableName(TABLE_NAME);
    dumpTable.setReflectCols(employeesCols);
    DataDumpers dataDumpers = dataSourceFactory.getDataDumpers(dumpTable);
    assertEquals(1, dataDumpers.splitCount);
    Iterator<IDataSourceDumper> dumpers = dataDumpers.dumpers;
    Map<String, Object> row = null;
    StringBuffer rowContent = null;
    int rowCount = 0;
    while (dumpers.hasNext()) {
        IDataSourceDumper dumper = dumpers.next();
        // assertEquals(300024, );
        assertTrue(dumper.getRowSize() > 0);
        try {
            Iterator<Map<String, Object>> rowIterator = dumper.startDump();
            while (rowIterator.hasNext()) {
                rowContent = new StringBuffer();
                row = rowIterator.next();
                // }
                for (Map.Entry<String, Object> entry : row.entrySet()) {
                    rowContent.append(entry.getKey()).append(":").append(entry.getValue()).append(",");
                }
                rowCount++;
                System.out.println(rowContent);
            }
        } finally {
            dumper.closeResource();
        }
    }
    assertEquals(300024, rowCount);
}

Example 2 with TISTable

use of com.qlangtech.tis.plugin.ds.TISTable in project plugins by qlangtech.

the class TestDorisSourceFactory method testDataDumpers.

public void testDataDumpers() throws Exception {
    DorisSourceFactory dataSourceFactory = getDorisSourceFactory();
    TISTable dumpTable = new TISTable();
    dumpTable.setSelectSql("SELECT emp_no,birth_date,first_name,last_name,gender,hire_date FROM employees");
    dumpTable.setTableName("employees");
    DataDumpers dataDumpers = dataSourceFactory.getDataDumpers(dumpTable);
    assertNotNull("dataDumpers can not be null", dataDumpers);
    assertEquals(1, dataDumpers.splitCount);
    Iterator<IDataSourceDumper> dumpers = dataDumpers.dumpers;
    Map<String, String> row = null;
    assertTrue("must contain a dumper", dumpers.hasNext());
    IDataSourceDumper dumper = dumpers.next();
    assertNotNull(dumper);
    assertEquals("jdbc:mysql://192.168.28.202:9030/", dumper.getDbHost());
// dumper.closeResource();
}

Example 3 with TISTable

use of com.qlangtech.tis.plugin.ds.TISTable in project tis by qlangtech.

the class ERRules method createDefaultErRule.

/**
 * 使用默认DumpNode创建ERRule并且持久化
 *
 * @param topology
 * @throws Exception
 */
public static void createDefaultErRule(SqlTaskNodeMeta.SqlDataFlowTopology topology) throws Exception {
    // 还没有定义erRule
    DependencyNode dumpNode = topology.getFirstDumpNode();
    DataSourceFactoryPluginStore dsStore = TIS.getDataBasePluginStore(new PostedDSProp(dumpNode.getDbName()));
    TISTable tab = dsStore.loadTableMeta(dumpNode.getName());
    // String topologyName, DependencyNode node, TargetColumnMeta targetColMetas
    Optional<ColumnMetaData> firstPK = tab.getReflectCols().stream().filter((col) -> col.isPk()).findFirst();
    if (!firstPK.isPresent()) {
        throw new IllegalStateException("table:" + dumpNode.parseEntityName() + " can not find relevant PK cols");
    }
    createErRule(topology.getName(), dumpNode, firstPK.get());
}

Example 4 with TISTable

use of com.qlangtech.tis.plugin.ds.TISTable in project plugins by qlangtech.

the class TisDataXTiDBReader method getTiKVDataSource.

private static final List<IDataSourceDumper> getTiKVDataSource(Configuration config, Optional<Long> regionId) {
    List<String> cols = config.getList("column", String.class);
    String tableName = config.getString("table");
    Configuration connection = config.getConfiguration("connection");
    TiKVDataSourceFactory sourceFactory = new TiKVDataSourceFactory();
    sourceFactory.pdAddrs = connection.getString("[0].pdAddrs");
    sourceFactory.dbName = connection.getString("[0].dbName");
    // sourceFactory.datetimeFormat = connection.getBool("datetimeFormat");
    if (StringUtils.isBlank(sourceFactory.pdAddrs) || StringUtils.isBlank(sourceFactory.dbName)) {
        throw new IllegalStateException("param 'pdAddrs' or 'dbName' can not be null,connection:" + connection.toJSON());
    }
    List<ColumnMetaData> tableMetadata = sourceFactory.getTableMetadata(tableName);
    TISTable table = new TISTable();
    table.setTableName(tableName);
    table.setReflectCols(tableMetadata.stream().filter((cmeta) -> cols.contains(cmeta.getKey())).collect(Collectors.toList()));
    DataDumpers dataDumpers = sourceFactory.getDataDumpers(table, regionId);
    List<IDataSourceDumper> dumpers = Lists.newArrayList(dataDumpers.dumpers);
    return dumpers;
}

Example 5 with TISTable

use of com.qlangtech.tis.plugin.ds.TISTable in project tis by qlangtech.

the class SingleTableAppSource method reflectCols.

@Override
public List<ColumnMetaData> reflectCols() {
    DataSourceFactoryPluginStore dataBasePluginStore = TIS.getDataBasePluginStore(new PostedDSProp(db.getName()));
    TISTable table = dataBasePluginStore.loadTableMeta(tabName);
    // });
    return table.getReflectCols();
}