Examples with SelectedTab com.qlangtech.tis.plugin.datax.SelectedTab used on opensource projects

Example 6 with SelectedTab

use of com.qlangtech.tis.plugin.datax.SelectedTab in project plugins by qlangtech.

the class BasicDataXRdbmsReader method getSubTasks.

// @Override
// public DBConfig getDbConfig() {
// return getBasicDataSource().getDbConfig();
// }
// 
// @Override
// public BasicDataSourceFactory getBasicDataSource() {
// return (BasicDataSourceFactory) getDataSourceFactory();
// }
@Override
public final IGroupChildTaskIterator getSubTasks() {
    Objects.requireNonNull(this.selectedTabs, "selectedTabs can not be null");
    DS dsFactory = this.getDataSourceFactory();
    Memoizer<String, Map<String, ColumnMetaData>> tabColsMap = getTabsMeta();
    AtomicInteger selectedTabIndex = new AtomicInteger(0);
    AtomicInteger taskIndex = new AtomicInteger(0);
    final int selectedTabsSize = this.selectedTabs.size();
    ConcurrentHashMap<String, List<String>> groupedInfo = new ConcurrentHashMap();
    AtomicReference<Iterator<IDataSourceDumper>> dumperItRef = new AtomicReference<>();
    return new IGroupChildTaskIterator() {

        @Override
        public Map<String, List<String>> getGroupedInfo() {
            return groupedInfo;
        }

        @Override
        public boolean hasNext() {
            Iterator<IDataSourceDumper> dumperIt = initDataSourceDumperIterator();
            if (dumperIt.hasNext()) {
                return true;
            } else {
                if (selectedTabIndex.get() >= selectedTabsSize) {
                    return false;
                } else {
                    dumperItRef.set(null);
                    initDataSourceDumperIterator();
                    return true;
                }
            }
        }

        private Iterator<IDataSourceDumper> initDataSourceDumperIterator() {
            Iterator<IDataSourceDumper> dumperIt;
            if ((dumperIt = dumperItRef.get()) == null) {
                SelectedTab tab = selectedTabs.get(selectedTabIndex.getAndIncrement());
                if (StringUtils.isEmpty(tab.getName())) {
                    throw new IllegalStateException("tableName can not be null");
                }
                // List<ColumnMetaData> tableMetadata = null;
                // IDataSourceDumper dumper = null;
                DataDumpers dataDumpers = null;
                TISTable tisTab = new TISTable();
                tisTab.setTableName(tab.getName());
                int[] index = { 0 };
                tisTab.setReflectCols(tab.getCols().stream().map((c) -> {
                    return createColumnMetaData(index, c.getName());
                }).collect(Collectors.toList()));
                dataDumpers = dsFactory.getDataDumpers(tisTab);
                dumperIt = dataDumpers.dumpers;
                dumperItRef.set(dumperIt);
            }
            return dumperIt;
        }

        @Override
        public IDataxReaderContext next() {
            Iterator<IDataSourceDumper> dumperIterator = dumperItRef.get();
            Objects.requireNonNull(dumperIterator, "dumperIterator can not be null,selectedTabIndex:" + selectedTabIndex.get());
            IDataSourceDumper dumper = dumperIterator.next();
            SelectedTab tab = selectedTabs.get(selectedTabIndex.get() - 1);
            String childTask = tab.getName() + "_" + taskIndex.getAndIncrement();
            List<String> childTasks = groupedInfo.computeIfAbsent(tab.getName(), (tabname) -> Lists.newArrayList());
            childTasks.add(childTask);
            RdbmsReaderContext dataxContext = createDataXReaderContext(childTask, tab, dumper);
            dataxContext.setWhere(tab.getWhere());
            if (isFilterUnexistCol()) {
                Map<String, ColumnMetaData> tableMetadata = tabColsMap.get(tab.getName());
                dataxContext.setCols(tab.cols.stream().filter((c) -> tableMetadata.containsKey(c)).collect(Collectors.toList()));
            } else {
                dataxContext.setCols(tab.cols);
            }
            return dataxContext;
        }
    };
}

Example 7 with SelectedTab

use of com.qlangtech.tis.plugin.datax.SelectedTab in project plugins by qlangtech.

the class TestSelectedTabs method createSelectedTabs.

public static List<SelectedTab> createSelectedTabs(int count) {
    List<SelectedTab> selectedTabs = Lists.newArrayList();
    SelectedTab selectedTab = new SelectedTab();
    selectedTab.setCols(Lists.newArrayList("col1", "col2", "col3"));
    selectedTab.setWhere("delete = 0");
    selectedTab.name = tabNameOrderDetail;
    selectedTabs.add(selectedTab);
    if (count > 1) {
        selectedTab = new SelectedTab();
        selectedTab.setCols(Lists.newArrayList("col1", "col2", "col3", "col4"));
        selectedTab.setWhere("delete = 0");
        selectedTab.name = tabNameTotalpayinfo;
        selectedTabs.add(selectedTab);
    }
    return selectedTabs;
}

Example 8 with SelectedTab

use of com.qlangtech.tis.plugin.datax.SelectedTab in project plugins by qlangtech.

the class TestTiKVDataSourceFactory method createTabOfEmployees.

public static List<SelectedTab> createTabOfEmployees() {
    List<SelectedTab> selectedTabs = Lists.newArrayList();
    SelectedTab selectedTab = new SelectedTab();
    selectedTab.setCols(Lists.newArrayList(COL_EMP_NO, COL_BIRTH_DATE, COL_FIRST_NAME, COL_LAST_NAME, COL_GENDER, COL_HIRE_DATE));
    selectedTab.name = TABLE_NAME;
    selectedTabs.add(selectedTab);
    return selectedTabs;
}